Fix legitimate database dump command errors (exit code 1) not being treated as errors by borgmatic (#310).
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Dan Helfman 2020-05-14 22:38:38 -07:00
parent 74adac6c70
commit d88f321cef
29 changed files with 174 additions and 167 deletions

6
NEWS
View File

@ -1,3 +1,9 @@
1.5.4.dev0
* #310: Fix legitimate database dump command errors (exit code 1) not being treated as errors by
borgmatic.
* For database dumps, replace the named pipe on every borgmatic run. This prevent hangs on stale
pipes left over from previous runs.
1.5.3
* #258: Stream database dumps and restores directly to/from Borg without using any additional
filesystem space. This feature is automatic, and works even on restores from archives made with

View File

@ -134,9 +134,9 @@ def check_archives(
# The Borg repair option trigger an interactive prompt, which won't work when output is
# captured. And progress messes with the terminal directly.
if repair or progress:
execute_command(full_command, output_file=DO_NOT_CAPTURE, error_on_warnings=True)
execute_command(full_command, output_file=DO_NOT_CAPTURE)
else:
execute_command(full_command, error_on_warnings=True)
execute_command(full_command)
if 'extract' in checks:
extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

View File

@ -215,7 +215,11 @@ def create_archive(
if stream_processes:
return execute_command_with_processes(
full_command, stream_processes, output_log_level, output_file, error_on_warnings=False
full_command,
stream_processes,
output_log_level,
output_file,
borg_local_path=local_path,
)
return execute_command(full_command, output_log_level, output_file, error_on_warnings=False)
return execute_command(full_command, output_log_level, output_file, borg_local_path=local_path)

View File

@ -28,7 +28,9 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
+ (repository,)
)
list_output = execute_command(full_list_command, output_log_level=None, error_on_warnings=False)
list_output = execute_command(
full_list_command, output_log_level=None, borg_local_path=local_path
)
try:
last_archive_name = list_output.strip().splitlines()[-1]
@ -49,7 +51,7 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
)
)
execute_command(full_extract_command, working_directory=None, error_on_warnings=True)
execute_command(full_extract_command, working_directory=None)
def extract_archive(
@ -63,7 +65,6 @@ def extract_archive(
remote_path=None,
destination_path=None,
progress=False,
error_on_warnings=True,
extract_to_stdout=False,
):
'''
@ -100,10 +101,7 @@ def extract_archive(
# the terminal directly.
if progress:
return execute_command(
full_command,
output_file=DO_NOT_CAPTURE,
working_directory=destination_path,
error_on_warnings=error_on_warnings,
full_command, output_file=DO_NOT_CAPTURE, working_directory=destination_path
)
return None
@ -112,12 +110,9 @@ def extract_archive(
full_command,
output_file=subprocess.PIPE,
working_directory=destination_path,
error_on_warnings=error_on_warnings,
run_to_completion=False,
)
# Error on warnings by default, as Borg only gives a warning if the restore paths don't exist in
# the archive!
execute_command(
full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
)
# Don't give Borg local path, so as to error on warnings, as Borg only gives a warning if the
# restore paths don't exist in the archive!
execute_command(full_command, working_directory=destination_path)

View File

@ -41,5 +41,5 @@ def display_archives_info(
return execute_command(
full_command,
output_log_level=None if info_arguments.json else logging.WARNING,
error_on_warnings=False,
borg_local_path=local_path,
)

View File

@ -55,4 +55,4 @@ def initialize_repository(
)
# Do not capture output here, so as to support interactive prompts.
execute_command(init_command, output_file=DO_NOT_CAPTURE, error_on_warnings=False)
execute_command(init_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)

View File

@ -36,7 +36,7 @@ def resolve_archive_name(repository, archive, storage_config, local_path='borg',
+ ('--short', repository)
)
output = execute_command(full_command, output_log_level=None, error_on_warnings=False)
output = execute_command(full_command, output_log_level=None, borg_local_path=local_path)
try:
latest_archive = output.strip().splitlines()[-1]
except IndexError:
@ -85,5 +85,5 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
return execute_command(
full_command,
output_log_level=None if list_arguments.json else logging.WARNING,
error_on_warnings=False,
borg_local_path=local_path,
)

View File

@ -40,7 +40,7 @@ def mount_archive(
# Don't capture the output when foreground mode is used so that ctrl-C can work properly.
if foreground:
execute_command(full_command, output_file=DO_NOT_CAPTURE, error_on_warnings=False)
execute_command(full_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)
return
execute_command(full_command, error_on_warnings=False)
execute_command(full_command, borg_local_path=local_path)

View File

@ -72,4 +72,4 @@ def prune_archives(
else:
output_log_level = logging.INFO
execute_command(full_command, output_log_level=output_log_level, error_on_warnings=False)
execute_command(full_command, output_log_level=output_log_level, borg_local_path=local_path)

View File

@ -17,4 +17,4 @@ def unmount_archive(mount_point, local_path='borg'):
+ (mount_point,)
)
execute_command(full_command, error_on_warnings=True)
execute_command(full_command)

View File

@ -11,15 +11,18 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25
BORG_ERROR_EXIT_CODE = 2
def exit_code_indicates_error(exit_code, error_on_warnings=True):
def exit_code_indicates_error(process, exit_code, borg_local_path=None):
'''
Return True if the given exit code from running a command corresponds to an error. If error on
warnings is False, then treat exit code 1 as a warning instead of an error.
Return True if the given exit code from running a command corresponds to an error. If a Borg
local path is given and matches the process' command, then treat exit code 1 as a warning
instead of an error.
'''
if error_on_warnings:
return bool(exit_code != 0)
command = process.args.split(' ') if isinstance(process.args, str) else process.args
return bool(exit_code >= BORG_ERROR_EXIT_CODE)
if borg_local_path and command[0] == borg_local_path:
return bool(exit_code >= BORG_ERROR_EXIT_CODE)
return bool(exit_code != 0)
def command_for_process(process):
@ -39,11 +42,11 @@ def output_buffer_for_process(process, exclude_stdouts):
return process.stderr if process.stdout in exclude_stdouts else process.stdout
def log_outputs(processes, exclude_stdouts, output_log_level, error_on_warnings):
def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
'''
Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
process with the requested log level. Additionally, raise a CalledProcessError if a process
exits with an error (or a warning, if error on warnings is True).
exits with an error (or a warning for exit code 1, if that process matches the Borg local path).
For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
stdouts are given to exclude, then for any matching processes, log from their stderr instead.
@ -99,7 +102,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, error_on_warnings)
for process in processes:
exit_code = process.wait()
if exit_code_indicates_error(exit_code, error_on_warnings):
if exit_code_indicates_error(process, exit_code, borg_local_path):
# If an error occurs, include its output in the raised exception so that we don't
# inadvertently hide error output.
output_buffer = output_buffer_for_process(process, exclude_stdouts)
@ -139,7 +142,7 @@ def execute_command(
shell=False,
extra_environment=None,
working_directory=None,
error_on_warnings=True,
borg_local_path=None,
run_to_completion=True,
):
'''
@ -150,9 +153,9 @@ def execute_command(
then read stdin from the file. If shell is True, execute the command within a shell. If an extra
environment dict is given, then use it to augment the current environment, and pass the result
into the command. If a working directory is given, use that as the present working directory
when running the command. If error on warnings is False, then treat exit code 1 as a warning
instead of an error. If run to completion is False, then return the process for the command
without executing it to completion.
when running the command. If a Borg local path is given, and the command matches it (regardless
of arguments), treat exit code 1 as a warning instead of an error. If run to completion is
False, then return the process for the command without executing it to completion.
Raise subprocesses.CalledProcessError if an error occurs while running the command.
'''
@ -179,7 +182,9 @@ def execute_command(
if not run_to_completion:
return process
log_outputs((process,), (input_file, output_file), output_log_level, error_on_warnings)
log_outputs(
(process,), (input_file, output_file), output_log_level, borg_local_path=borg_local_path
)
def execute_command_with_processes(
@ -191,7 +196,7 @@ def execute_command_with_processes(
shell=False,
extra_environment=None,
working_directory=None,
error_on_warnings=True,
borg_local_path=None,
):
'''
Execute the given command (a sequence of command/argument strings) and log its output at the
@ -204,8 +209,8 @@ def execute_command_with_processes(
the file. If shell is True, execute the command within a shell. If an extra environment dict is
given, then use it to augment the current environment, and pass the result into the command. If
a working directory is given, use that as the present working directory when running the
command. If error on warnings is False, then treat exit code 1 as a warning instead of an
error.
command. If a Borg local path is given, then for any matching command or process (regardless of
arguments), treat exit code 1 as a warning instead of an error.
Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
upstream process.
@ -240,5 +245,5 @@ def execute_command_with_processes(
tuple(processes) + (command_process,),
(input_file, output_file),
output_log_level,
error_on_warnings,
borg_local_path=borg_local_path,
)

View File

@ -38,8 +38,10 @@ def create_named_pipe_for_dump(dump_path):
Create a named pipe at the given dump path.
'''
os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
if not os.path.exists(dump_path):
os.mkfifo(dump_path, mode=0o600)
if os.path.exists(dump_path):
os.remove(dump_path)
os.mkfifo(dump_path, mode=0o600)
def remove_database_dumps(dump_path, databases, database_type_name, log_prefix, dry_run):

View File

@ -172,4 +172,5 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=extra_environment,
borg_local_path=location_config.get('local_path', 'borg'),
)

View File

@ -145,5 +145,6 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=extra_environment,
borg_local_path=location_config.get('local_path', 'borg'),
)
execute_command(analyze_command, extra_environment=extra_environment)

View File

@ -1,6 +1,6 @@
from setuptools import find_packages, setup
VERSION = '1.5.3'
VERSION = '1.5.4.dev0'
setup(

View File

@ -26,7 +26,7 @@ def test_log_outputs_logs_each_line_separately():
(hi_process, there_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
error_on_warnings=False,
borg_local_path='borg',
)
@ -49,7 +49,7 @@ def test_log_outputs_skips_logs_for_process_with_none_stdout():
(hi_process, there_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
error_on_warnings=False,
borg_local_path='borg',
)
@ -63,7 +63,7 @@ def test_log_outputs_includes_error_output_in_exception():
with pytest.raises(subprocess.CalledProcessError) as error:
module.log_outputs(
(process,), exclude_stdouts=(), output_log_level=logging.INFO, error_on_warnings=False
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
)
assert error.value.returncode == 2
@ -80,7 +80,7 @@ def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdou
with pytest.raises(subprocess.CalledProcessError) as error:
module.log_outputs(
(process,), exclude_stdouts=(), output_log_level=logging.INFO, error_on_warnings=False
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
)
assert error.value.returncode == 2
@ -98,7 +98,7 @@ def test_log_outputs_truncates_long_error_output():
with pytest.raises(subprocess.CalledProcessError) as error:
module.log_outputs(
(process,), exclude_stdouts=(), output_log_level=logging.INFO, error_on_warnings=False
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
)
assert error.value.returncode == 2
@ -113,5 +113,5 @@ def test_log_outputs_with_no_output_logs_nothing():
flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
module.log_outputs(
(process,), exclude_stdouts=(), output_log_level=logging.INFO, error_on_warnings=False
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
)

View File

@ -9,9 +9,7 @@ from ..test_verbosity import insert_logging_mock
def insert_execute_command_mock(command):
flexmock(module).should_receive('execute_command').with_args(
command, error_on_warnings=True
).once()
flexmock(module).should_receive('execute_command').with_args(command).once()
def insert_execute_command_never():
@ -165,9 +163,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter():
flexmock(module).should_receive('_make_check_flags').and_return(())
flexmock(module).should_receive('execute_command').never()
flexmock(module).should_receive('execute_command').with_args(
('borg', 'check', '--progress', 'repo'),
output_file=module.DO_NOT_CAPTURE,
error_on_warnings=True,
('borg', 'check', '--progress', 'repo'), output_file=module.DO_NOT_CAPTURE
).once()
module.check_archives(
@ -182,9 +178,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter():
flexmock(module).should_receive('_make_check_flags').and_return(())
flexmock(module).should_receive('execute_command').never()
flexmock(module).should_receive('execute_command').with_args(
('borg', 'check', '--repair', 'repo'),
output_file=module.DO_NOT_CAPTURE,
error_on_warnings=True,
('borg', 'check', '--repair', 'repo'), output_file=module.DO_NOT_CAPTURE
).once()
module.check_archives(

View File

@ -223,7 +223,7 @@ def test_create_archive_calls_borg_with_parameters():
('borg', 'create') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -252,7 +252,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
('borg', 'create') + pattern_flags + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -281,7 +281,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
('borg', 'create') + exclude_flags + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -308,7 +308,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
('borg', 'create', '--info') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.INFO)
@ -336,7 +336,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
output_log_level=None,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.INFO)
@ -364,7 +364,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
('borg', 'create', '--debug', '--show-rc') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.DEBUG)
@ -391,7 +391,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
output_log_level=None,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.DEBUG)
@ -420,7 +420,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
('borg', 'create', '--dry-run') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -449,7 +449,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
('borg', 'create', '--info', '--dry-run') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.INFO)
@ -477,7 +477,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
('borg', 'create', '--checkpoint-interval', '600') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -503,7 +503,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
('borg', 'create', '--chunker-params', '1,2,3,4') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -529,7 +529,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
('borg', 'create', '--compression', 'rle') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -555,7 +555,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
('borg', 'create', '--remote-ratelimit', '100') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -581,7 +581,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
('borg', 'create', '--one-file-system') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -608,7 +608,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
('borg', 'create', '--numeric-owner') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -635,7 +635,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -663,7 +663,7 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
('borg', 'create') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -691,7 +691,7 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
('borg', 'create', '--no' + option_name.replace('_', '')) + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -718,7 +718,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
('borg', 'create', '--files-cache', 'ctime,size') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -745,7 +745,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
('borg1', 'create') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg1',
)
module.create_archive(
@ -772,7 +772,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
('borg', 'create', '--remote-path', 'borg1') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -799,7 +799,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
('borg', 'create', '--umask', '740') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -825,7 +825,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
('borg', 'create', '--lock-wait', '5') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -851,7 +851,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
('borg', 'create', '--stats') + ARCHIVE_WITH_PATHS,
output_log_level=logging.WARNING,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -878,7 +878,7 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
('borg', 'create', '--info', '--stats') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.INFO)
@ -906,7 +906,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
('borg', 'create', '--list', '--filter', 'AME-') + ARCHIVE_WITH_PATHS,
output_log_level=logging.WARNING,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -933,7 +933,7 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
('borg', 'create', '--list', '--filter', 'AME-', '--info') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.INFO)
@ -961,7 +961,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
('borg', 'create', '--info', '--progress') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=module.DO_NOT_CAPTURE,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.INFO)
@ -989,7 +989,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
('borg', 'create', '--progress') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=module.DO_NOT_CAPTURE,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -1018,7 +1018,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
processes=processes,
output_log_level=logging.INFO,
output_file=module.DO_NOT_CAPTURE,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -1046,7 +1046,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
output_log_level=None,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
).and_return('[]')
json_output = module.create_archive(
@ -1075,7 +1075,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
output_log_level=None,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
).and_return('[]')
json_output = module.create_archive(
@ -1105,7 +1105,7 @@ def test_create_archive_with_source_directories_glob_expands():
('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
@ -1132,7 +1132,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'),
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
@ -1159,7 +1159,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -1185,7 +1185,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'),
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -1211,7 +1211,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'),
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -1237,7 +1237,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
('borg', 'create', '--extra', '--options') + ARCHIVE_WITH_PATHS,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(
@ -1265,7 +1265,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes():
processes=processes,
output_log_level=logging.INFO,
output_file=None,
error_on_warnings=False,
borg_local_path='borg',
)
module.create_archive(

View File

@ -8,15 +8,15 @@ from borgmatic.borg import extract as module
from ..test_verbosity import insert_logging_mock
def insert_execute_command_mock(command, working_directory=None, error_on_warnings=True):
def insert_execute_command_mock(command, working_directory=None):
flexmock(module).should_receive('execute_command').with_args(
command, working_directory=working_directory, error_on_warnings=error_on_warnings
command, working_directory=working_directory
).once()
def insert_execute_command_output_mock(command, result):
flexmock(module).should_receive('execute_command').with_args(
command, output_log_level=None, error_on_warnings=False
command, output_log_level=None, borg_local_path=command[0]
).and_return(result).once()
@ -226,7 +226,6 @@ def test_extract_archive_calls_borg_with_progress_parameter():
('borg', 'extract', '--progress', 'repo::archive'),
output_file=module.DO_NOT_CAPTURE,
working_directory=None,
error_on_warnings=True,
).once()
module.extract_archive(
@ -263,7 +262,6 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
('borg', 'extract', '--stdout', 'repo::archive'),
output_file=module.subprocess.PIPE,
working_directory=None,
error_on_warnings=True,
run_to_completion=False,
).and_return(process).once()
@ -284,7 +282,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
def test_extract_archive_skips_abspath_for_remote_repository():
flexmock(module.os.path).should_receive('abspath').never()
flexmock(module).should_receive('execute_command').with_args(
('borg', 'extract', 'server:repo::archive'), working_directory=None, error_on_warnings=True
('borg', 'extract', 'server:repo::archive'), working_directory=None
).once()
module.extract_archive(

View File

@ -10,7 +10,7 @@ from ..test_verbosity import insert_logging_mock
def test_display_archives_info_calls_borg_with_parameters():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
('borg', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
)
module.display_archives_info(
@ -20,9 +20,7 @@ def test_display_archives_info_calls_borg_with_parameters():
def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--info', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
('borg', 'info', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
)
insert_logging_mock(logging.INFO)
module.display_archives_info(
@ -32,7 +30,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
).and_return('[]')
insert_logging_mock(logging.INFO)
@ -47,7 +45,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--debug', '--show-rc', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.DEBUG)
@ -58,7 +56,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
).and_return('[]')
insert_logging_mock(logging.DEBUG)
@ -71,7 +69,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp
def test_display_archives_info_with_json_calls_borg_with_json_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
).and_return('[]')
json_output = module.display_archives_info(
@ -83,7 +81,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter():
def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING, error_on_warnings=False
('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
)
module.display_archives_info(
@ -93,7 +91,7 @@ def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
def test_display_archives_info_with_local_path_calls_borg_via_local_path():
flexmock(module).should_receive('execute_command').with_args(
('borg1', 'info', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
('borg1', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
)
module.display_archives_info(
@ -108,7 +106,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--remote-path', 'borg1', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
)
module.display_archives_info(
@ -124,7 +122,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--lock-wait', '5', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
)
module.display_archives_info(
@ -139,7 +137,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
)
module.display_archives_info(

View File

@ -24,7 +24,7 @@ def insert_info_command_not_found_mock():
def insert_init_command_mock(init_command, **kwargs):
flexmock(module).should_receive('execute_command').with_args(
init_command, output_file=module.DO_NOT_CAPTURE, error_on_warnings=False
init_command, output_file=module.DO_NOT_CAPTURE, borg_local_path=init_command[0]
).once()

View File

@ -26,9 +26,7 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name():
def test_resolve_archive_name_calls_borg_with_parameters():
expected_archive = 'archive-name'
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
output_log_level=None,
error_on_warnings=False,
('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
).and_return(expected_archive + '\n')
assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
@ -39,7 +37,7 @@ def test_resolve_archive_name_with_log_info_calls_borg_with_info_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--info') + BORG_LIST_LATEST_ARGUMENTS,
output_log_level=None,
error_on_warnings=False,
borg_local_path='borg',
).and_return(expected_archive + '\n')
insert_logging_mock(logging.INFO)
@ -51,7 +49,7 @@ def test_resolve_archive_name_with_log_debug_calls_borg_with_debug_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--debug', '--show-rc') + BORG_LIST_LATEST_ARGUMENTS,
output_log_level=None,
error_on_warnings=False,
borg_local_path='borg',
).and_return(expected_archive + '\n')
insert_logging_mock(logging.DEBUG)
@ -63,7 +61,7 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
flexmock(module).should_receive('execute_command').with_args(
('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS,
output_log_level=None,
error_on_warnings=False,
borg_local_path='borg1',
).and_return(expected_archive + '\n')
assert (
@ -77,7 +75,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS,
output_log_level=None,
error_on_warnings=False,
borg_local_path='borg',
).and_return(expected_archive + '\n')
assert (
@ -88,9 +86,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param
def test_resolve_archive_name_without_archives_raises():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
output_log_level=None,
error_on_warnings=False,
('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
).and_return('')
with pytest.raises(ValueError):
@ -103,7 +99,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS,
output_log_level=None,
error_on_warnings=False,
borg_local_path='borg',
).and_return(expected_archive + '\n')
assert (
@ -114,7 +110,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter
def test_list_archives_calls_borg_with_parameters():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
('borg', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
)
module.list_archives(
@ -126,9 +122,7 @@ def test_list_archives_calls_borg_with_parameters():
def test_list_archives_with_log_info_calls_borg_with_info_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--info', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
('borg', 'list', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
)
insert_logging_mock(logging.INFO)
@ -141,7 +135,7 @@ def test_list_archives_with_log_info_calls_borg_with_info_parameter():
def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
)
insert_logging_mock(logging.INFO)
@ -156,7 +150,7 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--debug', '--show-rc', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
)
insert_logging_mock(logging.DEBUG)
@ -169,7 +163,7 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
)
insert_logging_mock(logging.DEBUG)
@ -185,7 +179,7 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--lock-wait', '5', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
)
module.list_archives(
@ -198,7 +192,7 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
def test_list_archives_with_archive_calls_borg_with_archive_parameter():
storage_config = {}
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, error_on_warnings=False
('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
)
module.list_archives(
@ -213,7 +207,7 @@ def test_list_archives_with_path_calls_borg_with_path_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', 'repo::archive', 'var/lib'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
)
module.list_archives(
@ -225,7 +219,7 @@ def test_list_archives_with_path_calls_borg_with_path_parameter():
def test_list_archives_with_local_path_calls_borg_via_local_path():
flexmock(module).should_receive('execute_command').with_args(
('borg1', 'list', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
('borg1', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
)
module.list_archives(
@ -240,7 +234,7 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--remote-path', 'borg1', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
)
module.list_archives(
@ -255,7 +249,7 @@ def test_list_archives_with_short_calls_borg_with_short_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--short', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
).and_return('[]')
module.list_archives(
@ -283,7 +277,7 @@ def test_list_archives_passes_through_arguments_to_borg(argument_name):
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
).and_return('[]')
module.list_archives(
@ -299,7 +293,7 @@ def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--glob-archives', module.BORG_EXCLUDE_CHECKPOINTS_GLOB, 'repo'),
output_log_level=logging.WARNING,
error_on_warnings=False,
borg_local_path='borg',
).and_return('[]')
module.list_archives(
@ -311,7 +305,7 @@ def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
def test_list_archives_with_json_calls_borg_with_json_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
).and_return('[]')
json_output = module.list_archives(

View File

@ -9,7 +9,7 @@ from ..test_verbosity import insert_logging_mock
def insert_execute_command_mock(command):
flexmock(module).should_receive('execute_command').with_args(
command, error_on_warnings=False
command, borg_local_path='borg'
).once()
@ -120,7 +120,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter():
flexmock(module).should_receive('execute_command').with_args(
('borg', 'mount', '--foreground', 'repo::archive', '/mnt'),
output_file=module.DO_NOT_CAPTURE,
error_on_warnings=False,
borg_local_path='borg',
).once()
module.mount_archive(

View File

@ -10,7 +10,7 @@ from ..test_verbosity import insert_logging_mock
def insert_execute_command_mock(prune_command, output_log_level):
flexmock(module).should_receive('execute_command').with_args(
prune_command, output_log_level=output_log_level, error_on_warnings=False
prune_command, output_log_level=output_log_level, borg_local_path=prune_command[0]
).once()

View File

@ -8,9 +8,7 @@ from ..test_verbosity import insert_logging_mock
def insert_execute_command_mock(command):
flexmock(module).should_receive('execute_command').with_args(
command, error_on_warnings=True
).once()
flexmock(module).should_receive('execute_command').with_args(command).once()
def test_unmount_archive_calls_borg_with_required_parameters():

View File

@ -36,7 +36,8 @@ def test_make_database_dump_filename_with_invalid_name_raises():
def test_create_named_pipe_for_dump_does_not_raise():
flexmock(module.os).should_receive('makedirs')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module.os).should_receive('remove')
flexmock(module.os).should_receive('mkfifo')
module.create_named_pipe_for_dump('/path/to/pipe')

View File

@ -208,6 +208,7 @@ def test_restore_database_dump_runs_mysql_to_restore():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
borg_local_path='borg',
).once()
module.restore_database_dump(
@ -247,6 +248,7 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
borg_local_path='borg',
).once()
module.restore_database_dump(
@ -264,6 +266,7 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'MYSQL_PWD': 'trustsome1'},
borg_local_path='borg',
).once()
module.restore_database_dump(

View File

@ -208,6 +208,7 @@ def test_restore_database_dump_runs_pg_restore():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
borg_local_path='borg',
).once()
flexmock(module).should_receive('execute_command').with_args(
('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
@ -253,6 +254,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
borg_local_path='borg',
).once()
flexmock(module).should_receive('execute_command').with_args(
(
@ -296,6 +298,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'PGPASSWORD': 'trustsome1'},
borg_local_path='borg',
).once()
flexmock(module).should_receive('execute_command').with_args(
(
@ -327,6 +330,7 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
borg_local_path='borg',
).once()
flexmock(module).should_receive('execute_command').with_args(
('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None

View File

@ -7,23 +7,26 @@ from borgmatic import execute as module
@pytest.mark.parametrize(
'exit_code,error_on_warnings,expected_result',
'process,exit_code,borg_local_path,expected_result',
(
(2, True, True),
(2, False, True),
(1, True, True),
(1, False, False),
(0, True, False),
(0, False, False),
(flexmock(args=['grep']), 2, None, True),
(flexmock(args=['grep']), 2, 'borg', True),
(flexmock(args=['borg']), 2, 'borg', True),
(flexmock(args=['borg1']), 2, 'borg1', True),
(flexmock(args=['grep']), 1, None, True),
(flexmock(args=['grep']), 1, 'borg', True),
(flexmock(args=['borg']), 1, 'borg', False),
(flexmock(args=['borg1']), 1, 'borg1', False),
(flexmock(args=['grep']), 0, None, False),
(flexmock(args=['grep']), 0, 'borg', False),
(flexmock(args=['borg']), 0, 'borg', False),
(flexmock(args=['borg1']), 0, 'borg1', False),
),
)
def test_exit_code_indicates_error_respects_exit_code_and_error_on_warnings(
exit_code, error_on_warnings, expected_result
def test_exit_code_indicates_error_respects_exit_code_and_borg_local_path(
process, exit_code, borg_local_path, expected_result
):
assert (
module.exit_code_indicates_error(exit_code, error_on_warnings=error_on_warnings)
is expected_result
)
assert module.exit_code_indicates_error(process, exit_code, borg_local_path) is expected_result
def test_command_for_process_converts_sequence_command_to_string():