From 09183464cde4aa9d82a271c3cb19ac7ccabca1a8 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 22 Mar 2023 09:41:39 +0530 Subject: [PATCH 01/94] fix: no error on database backups without source dirs --- borgmatic/borg/create.py | 1 + borgmatic/execute.py | 32 +++++++++++++++++++++++--------- tests/unit/test_execute.py | 26 ++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 87a0fdd7..f0169bfc 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -291,6 +291,7 @@ def collect_special_file_paths( capture_stderr=True, working_directory=working_directory, extra_environment=borg_environment, + raise_on_exit_code_one=False, ) paths = tuple( diff --git a/borgmatic/execute.py b/borgmatic/execute.py index d4b04bfe..2daac555 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -213,7 +213,12 @@ def execute_command( def execute_command_and_capture_output( - full_command, capture_stderr=False, shell=False, extra_environment=None, working_directory=None, + full_command, + capture_stderr=False, + shell=False, + extra_environment=None, + working_directory=None, + raise_on_exit_code_one=True, ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its @@ -221,20 +226,29 @@ def execute_command_and_capture_output( stdout. If shell is True, execute the command within a shell. If an extra environment dict is given, then use it to augment the current environment, and pass the result into the command. If a working directory is given, use that as the present working directory when running the command. + If raise on exit code one is False, then treat exit code 1 as a warning instead of an error. - Raise subprocesses.CalledProcessError if an error occurs while running the command. + Raise subprocesses.CalledProcessError if an error occurs while running the command, or if the + command exits with a non-zero exit code and raise on exit code one is True. ''' log_command(full_command) environment = {**os.environ, **extra_environment} if extra_environment else None command = ' '.join(full_command) if shell else full_command - output = subprocess.check_output( - command, - stderr=subprocess.STDOUT if capture_stderr else None, - shell=shell, - env=environment, - cwd=working_directory, - ) + try: + output = subprocess.check_output( + command, + stderr=subprocess.STDOUT if capture_stderr else None, + shell=shell, + env=environment, + cwd=working_directory, + ) + logger.warning('Command output: {}'.format(output)) + except subprocess.CalledProcessError as e: + if raise_on_exit_code_one or e.returncode != 1: + raise + output = e.output + logger.warning('Command output: {}'.format(output)) return output.decode() if output is not None else None diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 0441e9d5..29a80cba 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -239,6 +239,32 @@ def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr() assert output == expected_output +def test_execute_command_and_capture_output_returns_output_with_raise_on_exit_code_one_false(): + full_command = ['foo', 'bar'] + expected_output = '[]' + err_output = b'[]' + flexmock(module.os, environ={'a': 'b'}) + flexmock(module.subprocess).should_receive('check_output').with_args( + full_command, stderr=None, shell=False, env=None, cwd=None + ).and_raise(subprocess.CalledProcessError(1, full_command, err_output)).once() + + output = module.execute_command_and_capture_output(full_command, raise_on_exit_code_one=False) + + assert output == expected_output + + +def test_execute_command_and_capture_output_returns_output_with_raise_on_exit_code_one_false_and_exit_code_not_one(): + full_command = ['foo', 'bar'] + expected_output = '[]' + flexmock(module.os, environ={'a': 'b'}) + flexmock(module.subprocess).should_receive('check_output').with_args( + full_command, stderr=None, shell=False, env=None, cwd=None + ).and_raise(subprocess.CalledProcessError(2, full_command, expected_output)).once() + + with pytest.raises(subprocess.CalledProcessError): + module.execute_command_and_capture_output(full_command, raise_on_exit_code_one=False) + + def test_execute_command_and_capture_output_returns_output_with_shell(): full_command = ['foo', 'bar'] expected_output = '[]' From bd235f042602a9a29cc5f81880e3b36c21c0fcc3 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 22 Mar 2023 16:23:53 +0530 Subject: [PATCH 02/94] use exit_code_indicates_error and modify it to accept a command --- borgmatic/borg/create.py | 2 +- borgmatic/execute.py | 20 +++++-------- tests/integration/test_execute.py | 8 ++--- tests/unit/test_execute.py | 50 ++++++++++++++++--------------- 4 files changed, 38 insertions(+), 42 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index f0169bfc..acc144b2 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -291,7 +291,7 @@ def collect_special_file_paths( capture_stderr=True, working_directory=working_directory, extra_environment=borg_environment, - raise_on_exit_code_one=False, + treat_exit_code_warning_as_error=False, ) paths = tuple( diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 2daac555..50a05879 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -11,7 +11,7 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25 BORG_ERROR_EXIT_CODE = 2 -def exit_code_indicates_error(process, exit_code, borg_local_path=None): +def exit_code_indicates_error(command, exit_code, borg_local_path=None): ''' Return True if the given exit code from running a command corresponds to an error. If a Borg local path is given and matches the process' command, then treat exit code 1 as a warning @@ -20,8 +20,6 @@ def exit_code_indicates_error(process, exit_code, borg_local_path=None): if exit_code is None: return False - command = process.args.split(' ') if isinstance(process.args, str) else process.args - if borg_local_path and command[0] == borg_local_path: return bool(exit_code < 0 or exit_code >= BORG_ERROR_EXIT_CODE) @@ -121,8 +119,9 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): if exit_code is None: still_running = True + command = process.args.split(' ') if isinstance(process.args, str) else process.args # If any process errors, then raise accordingly. - if exit_code_indicates_error(process, exit_code, borg_local_path): + if exit_code_indicates_error(command, exit_code, borg_local_path): # If an error occurs, include its output in the raised exception so that we don't # inadvertently hide error output. output_buffer = output_buffer_for_process(process, exclude_stdouts) @@ -213,12 +212,7 @@ def execute_command( def execute_command_and_capture_output( - full_command, - capture_stderr=False, - shell=False, - extra_environment=None, - working_directory=None, - raise_on_exit_code_one=True, + full_command, capture_stderr=False, shell=False, extra_environment=None, working_directory=None, ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its @@ -244,10 +238,10 @@ def execute_command_and_capture_output( cwd=working_directory, ) logger.warning('Command output: {}'.format(output)) - except subprocess.CalledProcessError as e: - if raise_on_exit_code_one or e.returncode != 1: + except subprocess.CalledProcessError as error: + if exit_code_indicates_error(error.returncode): raise - output = e.output + output = error.output logger.warning('Command output: {}'.format(output)) return output.decode() if output is not None else None diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index bbdb9771..2a9b61d7 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -138,10 +138,10 @@ def test_log_outputs_kills_other_processes_when_one_errors(): process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) flexmock(module).should_receive('exit_code_indicates_error').with_args( - process, None, 'borg' + ['grep'], None, 'borg' ).and_return(False) flexmock(module).should_receive('exit_code_indicates_error').with_args( - process, 2, 'borg' + ['grep'], 2, 'borg' ).and_return(True) other_process = subprocess.Popen( ['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT @@ -245,10 +245,10 @@ def test_log_outputs_truncates_long_error_output(): process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) flexmock(module).should_receive('exit_code_indicates_error').with_args( - process, None, 'borg' + ['grep'], None, 'borg' ).and_return(False) flexmock(module).should_receive('exit_code_indicates_error').with_args( - process, 2, 'borg' + ['grep'], 2, 'borg' ).and_return(True) flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout) diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 29a80cba..d6c51c0f 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -7,32 +7,32 @@ from borgmatic import execute as module @pytest.mark.parametrize( - 'process,exit_code,borg_local_path,expected_result', + 'command,exit_code,borg_local_path,expected_result', ( - (flexmock(args=['grep']), 2, None, True), - (flexmock(args=['grep']), 2, 'borg', True), - (flexmock(args=['borg']), 2, 'borg', True), - (flexmock(args=['borg1']), 2, 'borg1', True), - (flexmock(args=['grep']), 1, None, True), - (flexmock(args=['grep']), 1, 'borg', True), - (flexmock(args=['borg']), 1, 'borg', False), - (flexmock(args=['borg1']), 1, 'borg1', False), - (flexmock(args=['grep']), 0, None, False), - (flexmock(args=['grep']), 0, 'borg', False), - (flexmock(args=['borg']), 0, 'borg', False), - (flexmock(args=['borg1']), 0, 'borg1', False), + (['grep'], 2, None, True), + (['grep'], 2, 'borg', True), + (['borg'], 2, 'borg', True), + (['borg1'], 2, 'borg1', True), + (['grep'], 1, None, True), + (['grep'], 1, 'borg', True), + (['borg'], 1, 'borg', False), + (['borg1'], 1, 'borg1', False), + (['grep'], 0, None, False), + (['grep'], 0, 'borg', False), + (['borg'], 0, 'borg', False), + (['borg1'], 0, 'borg1', False), # -9 exit code occurs when child process get SIGKILLed. - (flexmock(args=['grep']), -9, None, True), - (flexmock(args=['grep']), -9, 'borg', True), - (flexmock(args=['borg']), -9, 'borg', True), - (flexmock(args=['borg1']), -9, 'borg1', True), - (flexmock(args=['borg']), None, None, False), + (['grep'], -9, None, True), + (['grep'], -9, 'borg', True), + (['borg'], -9, 'borg', True), + (['borg1'], -9, 'borg1', True), + (['borg'], None, None, False), ), ) def test_exit_code_indicates_error_respects_exit_code_and_borg_local_path( - process, exit_code, borg_local_path, expected_result + command, exit_code, borg_local_path, expected_result ): - assert module.exit_code_indicates_error(process, exit_code, borg_local_path) is expected_result + assert module.exit_code_indicates_error(command, exit_code, borg_local_path) is expected_result def test_command_for_process_converts_sequence_command_to_string(): @@ -239,7 +239,7 @@ def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr() assert output == expected_output -def test_execute_command_and_capture_output_returns_output_with_raise_on_exit_code_one_false(): +def test_execute_command_and_capture_output_returns_output_when_error_code_is_one(): full_command = ['foo', 'bar'] expected_output = '[]' err_output = b'[]' @@ -247,22 +247,24 @@ def test_execute_command_and_capture_output_returns_output_with_raise_on_exit_co flexmock(module.subprocess).should_receive('check_output').with_args( full_command, stderr=None, shell=False, env=None, cwd=None ).and_raise(subprocess.CalledProcessError(1, full_command, err_output)).once() + flexmock(module).should_receive('exit_code_indicates_error').and_return(False).once() - output = module.execute_command_and_capture_output(full_command, raise_on_exit_code_one=False) + output = module.execute_command_and_capture_output(full_command) assert output == expected_output -def test_execute_command_and_capture_output_returns_output_with_raise_on_exit_code_one_false_and_exit_code_not_one(): +def test_execute_command_and_capture_output_returns_output_when_error_code_not_one(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, stderr=None, shell=False, env=None, cwd=None ).and_raise(subprocess.CalledProcessError(2, full_command, expected_output)).once() + flexmock(module).should_receive('exit_code_indicates_error').and_return(True).once() with pytest.raises(subprocess.CalledProcessError): - module.execute_command_and_capture_output(full_command, raise_on_exit_code_one=False) + module.execute_command_and_capture_output(full_command) def test_execute_command_and_capture_output_returns_output_with_shell(): From 3b5ede8044fa93bd2f7bab40f1a6138af127d27b Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 22 Mar 2023 23:11:44 +0530 Subject: [PATCH 03/94] remove extra parameter from function call --- borgmatic/borg/create.py | 1 - 1 file changed, 1 deletion(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index acc144b2..87a0fdd7 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -291,7 +291,6 @@ def collect_special_file_paths( capture_stderr=True, working_directory=working_directory, extra_environment=borg_environment, - treat_exit_code_warning_as_error=False, ) paths = tuple( From 8a63c494984425f649375475b07215c9614598f6 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 23 Mar 2023 01:01:26 +0530 Subject: [PATCH 04/94] feat: tag repos --- borgmatic/actions/create.py | 8 +++---- borgmatic/commands/borgmatic.py | 18 ++++++++-------- borgmatic/config/normalize.py | 30 +++++++++++++++++++++----- borgmatic/config/schema.yaml | 37 +++++++++++++++++++++------------ borgmatic/config/validate.py | 11 ++++++---- 5 files changed, 69 insertions(+), 35 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 96a48521..1ef0ecab 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -42,11 +42,11 @@ def run_create( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) + logger.info('{}: Creating archive{}'.format(repository['path'], dry_run_label)) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, @@ -54,7 +54,7 @@ def run_create( active_dumps = borgmatic.hooks.dispatch.call_hooks( 'dump_databases', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, @@ -63,7 +63,7 @@ def run_create( json_output = borgmatic.borg.create.create_archive( global_arguments.dry_run, - repository, + repository['path'], location, storage, local_borg_version, diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fe07981e..1f8b15f4 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -108,7 +108,7 @@ def run_configuration(config_filename, config, arguments): repo_queue.put((repo, 0),) while not repo_queue.empty(): - repository_path, retry_num = repo_queue.get() + repository, retry_num = repo_queue.get() timeout = retry_num * retry_wait if timeout: logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') @@ -125,14 +125,14 @@ def run_configuration(config_filename, config, arguments): local_path=local_path, remote_path=remote_path, local_borg_version=local_borg_version, - repository_path=repository_path, + repository=repository, ) except (OSError, CalledProcessError, ValueError) as error: if retry_num < retries: - repo_queue.put((repository_path, retry_num + 1),) + repo_queue.put((repository, retry_num + 1),) tuple( # Consume the generator so as to trigger logging. log_error_records( - '{}: Error running actions for repository'.format(repository_path), + '{}: Error running actions for repository'.format(repository['path']), error, levelno=logging.WARNING, log_command_error_output=True, @@ -147,10 +147,10 @@ def run_configuration(config_filename, config, arguments): return yield from log_error_records( - '{}: Error running actions for repository'.format(repository_path), error + '{}: Error running actions for repository'.format(repository['path']), error ) encountered_error = error - error_repository = repository_path + error_repository = repository['path'] try: if using_primary_action: @@ -248,7 +248,7 @@ def run_actions( local_path, remote_path, local_borg_version, - repository_path, + repository, ): ''' Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration @@ -263,13 +263,13 @@ def run_actions( invalid. ''' add_custom_log_levels() - repository = os.path.expanduser(repository_path) + repository_path = os.path.expanduser(repository['path']) global_arguments = arguments['global'] dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' hook_context = { 'repository': repository_path, # Deprecated: For backwards compatibility with borgmatic < 1.6.0. - 'repositories': ','.join(location['repositories']), + 'repositories': ','.join([repo['path'] for repo in location['repositories']]), } command.execute_hook( diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index a143a192..58e34680 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -56,9 +56,13 @@ def normalize(config_filename, config): # Upgrade remote repositories to ssh:// syntax, required in Borg 2. repositories = location.get('repositories') + if isinstance(repositories[0], str): + config['location']['repositories'] = [{'path': repository} for repository in repositories] + repositories = config['location']['repositories'] if repositories: config['location']['repositories'] = [] - for repository in repositories: + for repository_dict in repositories: + repository = repository_dict['path'] if '~' in repository: logs.append( logging.makeLogRecord( @@ -71,11 +75,19 @@ def normalize(config_filename, config): ) if ':' in repository: if repository.startswith('file://'): + updated_repository_path = os.path.abspath(repository.partition('file://')[-1]) + config['location']['repositories'].append( - os.path.abspath(repository.partition('file://')[-1]) + { + 'path': updated_repository_path, + 'label': repository_dict.get('label', None), + } ) elif repository.startswith('ssh://'): - config['location']['repositories'].append(repository) + config['location']['repositories'].append({ + 'path': repository, + 'label': repository_dict.get('label', None), + }) else: rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( @@ -87,8 +99,16 @@ def normalize(config_filename, config): ) ) ) - config['location']['repositories'].append(rewritten_repository) + config['location']['repositories'].append({ + 'path': rewritten_repository, + 'label': repository_dict.get('label', None), + }) else: - config['location']['repositories'].append(repository) + config['location']['repositories'].append( + { + 'path': repository, + 'label': repository_dict.get('label', None), + } + ) return logs diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d4d57ab6..4235adba 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -29,19 +29,30 @@ properties: repositories: type: array items: - type: string - description: | - Paths to local or remote repositories (required). Tildes are - expanded. Multiple repositories are backed up to in - sequence. Borg placeholders can be used. See the output of - "borg help placeholders" for details. See ssh_command for - SSH options like identity file or port. If systemd service - is used, then add local repository paths in the systemd - service file to the ReadWritePaths list. - example: - - ssh://user@backupserver/./sourcehostname.borg - - ssh://user@backupserver/./{fqdn} - - /var/local/backups/local.borg + type: object + required: + - path + properties: + path: + type: string + description: | + Path to local or remote repository (required). + are expanded. Multiple repositories are backed up to + in sequence. Borg placeholders can be used. See the + output of "borg help placeholders" for details. See + ssh_command for SSH options like identity file or + port. If systemd service is used, then add local + repository paths in the systemd service file to the + ReadWritePaths list. + example: + - ssh://user@backupserver/./sourcehostname.borg + - ssh://user@backupserver/./{fqdn} + - /var/local/backups/local.borg + label: + type: string + description: | + Optional label for the repository. + example: backupserver working_directory: type: string description: | diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 5828380e..9b64377a 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -138,10 +138,13 @@ def normalize_repository_path(repository): def repositories_match(first, second): ''' - Given two repository paths (relative and/or absolute), return whether they match. + Given two repository dicts with keys 'path' (relative and/or absolute), + and 'label', return whether they match. ''' - return normalize_repository_path(first) == normalize_repository_path(second) - + if isinstance(first,str) and isinstance(second,str): + return normalize_repository_path(first) == normalize_repository_path(second) + elif isinstance(first,dict) and isinstance(second,str): + return (second == first.get('label')) or (normalize_repository_path(second) == normalize_repository_path(first.get('path'))) def guard_configuration_contains_repository(repository, configurations): ''' @@ -160,7 +163,7 @@ def guard_configuration_contains_repository(repository, configurations): config_repository for config in configurations.values() for config_repository in config['location']['repositories'] - if repositories_match(repository, config_repository) + if repositories_match(config_repository, repository) ) ) From 7a2f287918beea36378aa0dd9fbdf8ca9ae16355 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 23 Mar 2023 01:08:30 +0530 Subject: [PATCH 05/94] reformat base --- borgmatic/config/normalize.py | 19 +++++++------------ borgmatic/config/validate.py | 9 ++++++--- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 58e34680..8ce4af22 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -84,10 +84,9 @@ def normalize(config_filename, config): } ) elif repository.startswith('ssh://'): - config['location']['repositories'].append({ - 'path': repository, - 'label': repository_dict.get('label', None), - }) + config['location']['repositories'].append( + {'path': repository, 'label': repository_dict.get('label', None),} + ) else: rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( @@ -99,16 +98,12 @@ def normalize(config_filename, config): ) ) ) - config['location']['repositories'].append({ - 'path': rewritten_repository, - 'label': repository_dict.get('label', None), - }) + config['location']['repositories'].append( + {'path': rewritten_repository, 'label': repository_dict.get('label', None),} + ) else: config['location']['repositories'].append( - { - 'path': repository, - 'label': repository_dict.get('label', None), - } + {'path': repository, 'label': repository_dict.get('label', None),} ) return logs diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 9b64377a..b43d42df 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -141,10 +141,13 @@ def repositories_match(first, second): Given two repository dicts with keys 'path' (relative and/or absolute), and 'label', return whether they match. ''' - if isinstance(first,str) and isinstance(second,str): + if isinstance(first, str) and isinstance(second, str): return normalize_repository_path(first) == normalize_repository_path(second) - elif isinstance(first,dict) and isinstance(second,str): - return (second == first.get('label')) or (normalize_repository_path(second) == normalize_repository_path(first.get('path'))) + elif isinstance(first, dict) and isinstance(second, str): + return (second == first.get('label')) or ( + normalize_repository_path(second) == normalize_repository_path(first.get('path')) + ) + def guard_configuration_contains_repository(repository, configurations): ''' From 1e3a3bf1e7b73fa34e73e6905d6b17c27bb1bd5f Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 23 Mar 2023 01:18:06 +0530 Subject: [PATCH 06/94] review --- borgmatic/execute.py | 6 ++---- tests/unit/test_execute.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 50a05879..a01e1a13 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -220,10 +220,8 @@ def execute_command_and_capture_output( stdout. If shell is True, execute the command within a shell. If an extra environment dict is given, then use it to augment the current environment, and pass the result into the command. If a working directory is given, use that as the present working directory when running the command. - If raise on exit code one is False, then treat exit code 1 as a warning instead of an error. - Raise subprocesses.CalledProcessError if an error occurs while running the command, or if the - command exits with a non-zero exit code and raise on exit code one is True. + Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' log_command(full_command) environment = {**os.environ, **extra_environment} if extra_environment else None @@ -239,7 +237,7 @@ def execute_command_and_capture_output( ) logger.warning('Command output: {}'.format(output)) except subprocess.CalledProcessError as error: - if exit_code_indicates_error(error.returncode): + if exit_code_indicates_error(command, error.returncode): raise output = error.output logger.warning('Command output: {}'.format(output)) diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index d6c51c0f..3226a5a6 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -254,7 +254,7 @@ def test_execute_command_and_capture_output_returns_output_when_error_code_is_on assert output == expected_output -def test_execute_command_and_capture_output_returns_output_when_error_code_not_one(): +def test_execute_command_and_capture_output_raises_when_command_errors(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module.os, environ={'a': 'b'}) From 0a72c67c6c2704ea5e8dfa683facd1e8d423e138 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 22 Mar 2023 13:02:22 -0700 Subject: [PATCH 07/94] Add missing source directory error fix to NEWS (#655). --- NEWS | 1 + borgmatic/borg/rlist.py | 2 +- tests/unit/test_execute.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 0bb47e26..bd5687cd 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,7 @@ * #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in borgmatic's storage configuration. * #623: Fix confusing message when an error occurs running actions for a configuration file. + * #655: Fix error when databases are configured and a source directory doesn't exist. 1.7.9 * #295: Add a SQLite database dump/restore hook. diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 2a465fb0..33404f9e 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -117,7 +117,7 @@ def list_repository( ) if rlist_arguments.json: - return execute_command_and_capture_output(main_command, extra_environment=borg_environment,) + return execute_command_and_capture_output(main_command, extra_environment=borg_environment) else: execute_command( main_command, diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 3226a5a6..badcb8fd 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -239,7 +239,7 @@ def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr() assert output == expected_output -def test_execute_command_and_capture_output_returns_output_when_error_code_is_one(): +def test_execute_command_and_capture_output_returns_output_when_process_error_is_not_considered_an_error(): full_command = ['foo', 'bar'] expected_output = '[]' err_output = b'[]' From 66d2f49f18c4a39ccb03955b02ef82d2d09d0007 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 23 Mar 2023 14:45:23 +0530 Subject: [PATCH 08/94] docs: copy to clipboard support --- .eleventy.js | 3 +++ docs/Dockerfile | 1 + docs/_includes/layouts/base.njk | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.eleventy.js b/.eleventy.js index bde54af7..30057a4a 100644 --- a/.eleventy.js +++ b/.eleventy.js @@ -1,4 +1,5 @@ const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight"); +const codeClipboard = require("eleventy-plugin-code-clipboard"); const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language"); const navigationPlugin = require("@11ty/eleventy-navigation"); @@ -6,6 +7,7 @@ module.exports = function(eleventyConfig) { eleventyConfig.addPlugin(pluginSyntaxHighlight); eleventyConfig.addPlugin(inclusiveLangPlugin); eleventyConfig.addPlugin(navigationPlugin); + eleventyConfig.addPlugin(codeClipboard); let markdownIt = require("markdown-it"); let markdownItAnchor = require("markdown-it-anchor"); @@ -31,6 +33,7 @@ module.exports = function(eleventyConfig) { markdownIt(markdownItOptions) .use(markdownItAnchor, markdownItAnchorOptions) .use(markdownItReplaceLink) + .use(codeClipboard.markdownItCopyButton) ); eleventyConfig.addPassthroughCopy({"docs/static": "static"}); diff --git a/docs/Dockerfile b/docs/Dockerfile index 8800cc1f..dcda44f3 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -18,6 +18,7 @@ RUN npm install @11ty/eleventy \ @11ty/eleventy-plugin-syntaxhighlight \ @11ty/eleventy-plugin-inclusive-language \ @11ty/eleventy-navigation \ + eleventy-plugin-code-clipboard \ markdown-it \ markdown-it-anchor \ markdown-it-replace-link diff --git a/docs/_includes/layouts/base.njk b/docs/_includes/layouts/base.njk index 361967c5..61ae7a48 100644 --- a/docs/_includes/layouts/base.njk +++ b/docs/_includes/layouts/base.njk @@ -18,10 +18,12 @@ {% if feedTitle and feedUrl %} {% endif %} + + {{ content | safe }} - + {% initClipboardJS %} From a01dc62468e01e25f2e07f2e587a1b8266a3ac25 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 24 Mar 2023 01:23:40 +0530 Subject: [PATCH 09/94] fix: remove extra links from docs css --- docs/_includes/index.css | 15 +++++++++++++++ docs/_includes/layouts/base.njk | 2 -- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/_includes/index.css b/docs/_includes/index.css index ca1c2dfa..9d8c9558 100644 --- a/docs/_includes/index.css +++ b/docs/_includes/index.css @@ -533,3 +533,18 @@ main .elv-toc + h1 .direct-link { .header-anchor:hover::after { content: " 🔗"; } + +.mdi { + display: inline-block; + width: 1em; + height: 1em; + background-color: currentColor; + -webkit-mask: no-repeat center / 100%; + mask: no-repeat center / 100%; + -webkit-mask-image: var(--svg); + mask-image: var(--svg); +} + +.mdi.mdi-content-copy { + --svg: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' width='24' height='24'%3E%3Cpath fill='black' d='M19 21H8V7h11m0-2H8a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h11a2 2 0 0 0 2-2V7a2 2 0 0 0-2-2m-3-4H4a2 2 0 0 0-2 2v14h2V3h12V1Z'/%3E%3C/svg%3E"); +} \ No newline at end of file diff --git a/docs/_includes/layouts/base.njk b/docs/_includes/layouts/base.njk index 61ae7a48..be588e9b 100644 --- a/docs/_includes/layouts/base.njk +++ b/docs/_includes/layouts/base.njk @@ -18,8 +18,6 @@ {% if feedTitle and feedUrl %} {% endif %} - - From 08afad5d819468d315f2400f9c6affd4de1620be Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 24 Mar 2023 01:25:15 +0530 Subject: [PATCH 10/94] end with newline --- docs/_includes/index.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_includes/index.css b/docs/_includes/index.css index 9d8c9558..f1d4c57f 100644 --- a/docs/_includes/index.css +++ b/docs/_includes/index.css @@ -547,4 +547,4 @@ main .elv-toc + h1 .direct-link { .mdi.mdi-content-copy { --svg: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' width='24' height='24'%3E%3Cpath fill='black' d='M19 21H8V7h11m0-2H8a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h11a2 2 0 0 0 2-2V7a2 2 0 0 0-2-2m-3-4H4a2 2 0 0 0-2 2v14h2V3h12V1Z'/%3E%3C/svg%3E"); -} \ No newline at end of file +} From 6f300b00795dd76ced62407f9eb065778a8bcbe6 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 24 Mar 2023 02:39:37 +0530 Subject: [PATCH 11/94] feat: constants support --- borgmatic/config/load.py | 13 +++++++++++-- borgmatic/config/schema.yaml | 11 +++++++++++ tests/integration/config/test_load.py | 17 ++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 04461af0..e379537a 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -81,7 +81,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor): def load_configuration(filename): ''' Load the given configuration file and return its contents as a data structure of nested dicts - and lists. + and lists. Also, replace any "{constant}" strings with the value of the "constant" key in the + "constants" section of the configuration file. Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError if there are too many recursive includes. @@ -98,7 +99,15 @@ def load_configuration(filename): yaml = ruamel.yaml.YAML(typ='safe') yaml.Constructor = Include_constructor_with_include_directory - return yaml.load(open(filename)) + with open(filename) as f: + file_contents = f.read() + config = yaml.load(file_contents) + if config and 'constants' in config: + for key, value in config['constants'].items(): + file_contents = file_contents.replace(f'{{{key}}}', str(value)) + config = yaml.load(file_contents) + del config['constants'] + return config DELETED_NODE = object() diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d4d57ab6..c01f3db1 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -3,6 +3,17 @@ required: - location additionalProperties: false properties: + constants: + type: object + description: | + Constants to use in the configuration file. All occurences of the + constant name within culy braces will be replaced with the value. + For example, if you have a constant named "hostname" with the value + "myhostname", then the string "{hostname}" will be replaced with + "myhostname" in the configuration file. + example: + hostname: myhostname + prefix: myprefix location: type: object description: | diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index e1ecc8ae..5b8ee9b0 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -10,8 +10,23 @@ from borgmatic.config import load as module def test_load_configuration_parses_contents(): builtins = flexmock(sys.modules['builtins']) - builtins.should_receive('open').with_args('config.yaml').and_return('key: value') + config_file = io.StringIO('key: value') + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + assert module.load_configuration('config.yaml') == {'key': 'value'} + +def test_load_configuration_replaces_constants(): + builtins = flexmock(sys.modules['builtins']) + config_file = io.StringIO( + ''' + constants: + key: value + key: {key} + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) assert module.load_configuration('config.yaml') == {'key': 'value'} From f42890430c59a40a17d9a68a193d6a09674770cb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 23 Mar 2023 23:11:14 -0700 Subject: [PATCH 12/94] Add code style plugins to enforce use of Python f-strings and prevent single-letter variables. --- NEWS | 2 ++ borgmatic/actions/borg.py | 2 +- borgmatic/actions/check.py | 2 +- borgmatic/actions/compact.py | 4 +-- borgmatic/actions/create.py | 2 +- borgmatic/actions/export_tar.py | 4 +-- borgmatic/actions/extract.py | 2 +- borgmatic/actions/mount.py | 4 +-- borgmatic/actions/prune.py | 2 +- borgmatic/actions/rcreate.py | 2 +- borgmatic/actions/restore.py | 5 ++-- borgmatic/actions/rinfo.py | 3 +- borgmatic/actions/rlist.py | 3 +- borgmatic/borg/check.py | 4 +-- borgmatic/borg/create.py | 2 +- borgmatic/borg/export_tar.py | 2 +- borgmatic/borg/flags.py | 2 +- borgmatic/borg/list.py | 2 +- borgmatic/borg/prune.py | 2 +- borgmatic/borg/rlist.py | 2 +- borgmatic/commands/arguments.py | 4 +-- borgmatic/commands/borgmatic.py | 38 ++++++++++-------------- borgmatic/commands/completion.py | 4 +-- borgmatic/commands/convert_config.py | 22 +++++--------- borgmatic/commands/generate_config.py | 14 +++------ borgmatic/commands/validate_config.py | 10 ++----- borgmatic/config/collect.py | 4 +-- borgmatic/config/environment.py | 5 +++- borgmatic/config/generate.py | 10 +++---- borgmatic/config/legacy.py | 16 ++++------ borgmatic/config/validate.py | 23 +++++++------- borgmatic/execute.py | 8 ++--- borgmatic/hooks/command.py | 18 ++++------- borgmatic/hooks/cronhub.py | 8 ++--- borgmatic/hooks/cronitor.py | 8 ++--- borgmatic/hooks/dispatch.py | 4 +-- borgmatic/hooks/dump.py | 8 ++--- borgmatic/hooks/healthchecks.py | 10 +++---- borgmatic/hooks/mongodb.py | 10 ++----- borgmatic/hooks/mysql.py | 6 ++-- borgmatic/hooks/pagerduty.py | 10 +++---- borgmatic/hooks/postgresql.py | 6 ++-- borgmatic/hooks/sqlite.py | 2 +- borgmatic/logger.py | 2 +- test_requirements.txt | 2 ++ tests/end-to-end/test_borgmatic.py | 20 +++++-------- tests/end-to-end/test_database.py | 2 +- tests/end-to-end/test_override.py | 8 ++--- tests/end-to-end/test_validate_config.py | 16 +++------- tests/integration/config/test_legacy.py | 2 +- tests/unit/borg/test_create.py | 20 ++++++------- tests/unit/borg/test_prune.py | 22 ++++++++++---- tests/unit/config/test_environment.py | 28 ++++++++--------- tests/unit/config/test_validate.py | 8 ++--- tests/unit/hooks/test_command.py | 17 ++++------- tests/unit/hooks/test_healthchecks.py | 4 +-- tests/unit/hooks/test_mongodb.py | 2 +- tests/unit/hooks/test_postgresql.py | 2 +- 58 files changed, 195 insertions(+), 261 deletions(-) diff --git a/NEWS b/NEWS index bd5687cd..bc5f431e 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,8 @@ in borgmatic's storage configuration. * #623: Fix confusing message when an error occurs running actions for a configuration file. * #655: Fix error when databases are configured and a source directory doesn't exist. + * Add code style plugins to enforce use of Python f-strings and prevent single-letter variables. + To join in the pedantry, refresh your test environment with "tox --recreate". 1.7.9 * #295: Add a SQLite database dump/restore hook. diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index a50dd286..ee94f1dc 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -16,7 +16,7 @@ def run_borg( if borg_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, borg_arguments.repository ): - logger.info('{}: Running arbitrary Borg command'.format(repository)) + logger.info(f'{repository}: Running arbitrary Borg command') archive_name = borgmatic.borg.rlist.resolve_archive_name( repository, borg_arguments.archive, diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index f3572395..cdee9edb 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -37,7 +37,7 @@ def run_check( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Running consistency checks'.format(repository)) + logger.info(f'{repository}: Running consistency checks') borgmatic.borg.check.check_archives( repository, location, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 7a25b829..a0efa3a2 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -39,7 +39,7 @@ def run_compact( **hook_context, ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): - logger.info('{}: Compacting segments{}'.format(repository, dry_run_label)) + logger.info(f'{repository}: Compacting segments{dry_run_label}') borgmatic.borg.compact.compact_segments( global_arguments.dry_run, repository, @@ -52,7 +52,7 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover - logger.info('{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository)) + logger.info(f'{repository}: Skipping compact (only available/needed in Borg 1.2+)') borgmatic.hooks.command.execute_hook( hooks.get('after_compact'), hooks.get('umask'), diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 96a48521..ac2617d2 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -42,7 +42,7 @@ def run_create( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) + logger.info(f'{repository}: Creating archive{dry_run_label}') borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index ae349208..b5b6089d 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -22,9 +22,7 @@ def run_export_tar( if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, export_tar_arguments.repository ): - logger.info( - '{}: Exporting archive {} as tar file'.format(repository, export_tar_arguments.archive) - ) + logger.info(f'{repository}: Exporting archive {export_tar_arguments.archive} as tar file') borgmatic.borg.export_tar.export_tar_archive( global_arguments.dry_run, repository, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index a3d89a55..6af9caa1 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -35,7 +35,7 @@ def run_extract( if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): - logger.info('{}: Extracting archive {}'.format(repository, extract_arguments.archive)) + logger.info(f'{repository}: Extracting archive {extract_arguments.archive}') borgmatic.borg.extract.extract_archive( global_arguments.dry_run, repository, diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index e2703a57..262e7d9e 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -17,9 +17,9 @@ def run_mount( repository, mount_arguments.repository ): if mount_arguments.archive: - logger.info('{}: Mounting archive {}'.format(repository, mount_arguments.archive)) + logger.info(f'{repository}: Mounting archive {mount_arguments.archive}') else: # pragma: nocover - logger.info('{}: Mounting repository'.format(repository)) + logger.info(f'{repository}: Mounting repository') borgmatic.borg.mount.mount_archive( repository, diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index ca098ce4..76a42a9a 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -37,7 +37,7 @@ def run_prune( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Pruning archives{}'.format(repository, dry_run_label)) + logger.info(f'{repository}: Pruning archives{dry_run_label}') borgmatic.borg.prune.prune_archives( global_arguments.dry_run, repository, diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index 0052b4b6..59b147d7 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -23,7 +23,7 @@ def run_rcreate( ): return - logger.info('{}: Creating repository'.format(repository)) + logger.info(f'{repository}: Creating repository') borgmatic.borg.rcreate.create_repository( global_arguments.dry_run, repository, diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 7a058092..50c39737 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -255,9 +255,8 @@ def run_restore( ): return - logger.info( - '{}: Restoring databases from archive {}'.format(repository, restore_arguments.archive) - ) + logger.info(f'{repository}: Restoring databases from archive {restore_arguments.archive}') + borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 611d1bc2..e7132c04 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -19,7 +19,8 @@ def run_rinfo( repository, rinfo_arguments.repository ): if not rinfo_arguments.json: # pragma: nocover - logger.answer('{}: Displaying repository summary information'.format(repository)) + logger.answer(f'{repository}: Displaying repository summary information') + json_output = borgmatic.borg.rinfo.display_repository_info( repository, storage, diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index 72d52068..aa2032b1 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -19,7 +19,8 @@ def run_rlist( repository, rlist_arguments.repository ): if not rlist_arguments.json: # pragma: nocover - logger.answer('{}: Listing repository'.format(repository)) + logger.answer(f'{repository}: Listing repository') + json_output = borgmatic.borg.rlist.list_repository( repository, storage, diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index d9beaa60..2914d83b 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -12,7 +12,7 @@ DEFAULT_CHECKS = ( {'name': 'repository', 'frequency': '1 month'}, {'name': 'archives', 'frequency': '1 month'}, ) -DEFAULT_PREFIX = '{hostname}-' +DEFAULT_PREFIX = '{hostname}-' # noqa: FS003 logger = logging.getLogger(__name__) @@ -196,7 +196,7 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): return common_flags return ( - tuple('--{}-only'.format(check) for check in checks if check in ('repository', 'archives')) + tuple(f'--{check}-only' for check in checks if check in ('repository', 'archives')) + common_flags ) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 87a0fdd7..d557a6ab 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -217,7 +217,7 @@ def make_list_filter_flags(local_borg_version, dry_run): return f'{base_flags}-' -DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' +DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003 def collect_borgmatic_source_directories(borgmatic_source_directory): diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 43ea4ac0..01d1b7ed 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -56,7 +56,7 @@ def export_tar_archive( output_log_level = logging.INFO if dry_run: - logging.info('{}: Skipping export to tar file (dry run)'.format(repository)) + logging.info(f'{repository}: Skipping export to tar file (dry run)') return execute_command( diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 81b6a6b1..845e0ff3 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -10,7 +10,7 @@ def make_flags(name, value): if not value: return () - flag = '--{}'.format(name.replace('_', '-')) + flag = f"--{name.replace('_', '-')}" if value is True: return (flag,) diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index fedd3650..916d17b0 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -113,7 +113,7 @@ def capture_archive_listing( paths=[f'sh:{list_path}'], find_paths=None, json=None, - format='{path}{NL}', + format='{path}{NL}', # noqa: FS003 ), local_path, remote_path, diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index e53197f1..5be85de2 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -24,7 +24,7 @@ def make_prune_flags(retention_config, local_borg_version): ) ''' config = retention_config.copy() - prefix = config.pop('prefix', '{hostname}-') + prefix = config.pop('prefix', '{hostname}-') # noqa: FS003 if prefix: if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 33404f9e..43bc28d6 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -42,7 +42,7 @@ def resolve_archive_name( except IndexError: raise ValueError('No archives found in the repository') - logger.debug('{}: Latest archive is {}'.format(repository, latest_archive)) + logger.debug(f'{repository}: Latest archive is {latest_archive}') return latest_archive diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index d5dc6af4..773a0bae 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -131,9 +131,7 @@ def make_parsers(): nargs='*', dest='config_paths', default=config_paths, - help='Configuration filenames or directories, defaults to: {}'.format( - ' '.join(unexpanded_config_paths) - ), + help=f"Configuration filenames or directories, defaults to: {' '.join(unexpanded_config_paths)}", ) global_group.add_argument( '--excludes', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fbea260d..73bde94b 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -70,9 +70,7 @@ def run_configuration(config_filename, config, arguments): try: local_borg_version = borg_version.local_borg_version(storage, local_path) except (OSError, CalledProcessError, ValueError) as error: - yield from log_error_records( - '{}: Error getting local Borg version'.format(config_filename), error - ) + yield from log_error_records(f'{config_filename}: Error getting local Borg version', error) return try: @@ -100,7 +98,7 @@ def run_configuration(config_filename, config, arguments): return encountered_error = error - yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error) + yield from log_error_records(f'{config_filename}: Error pinging monitor', error) if not encountered_error: repo_queue = Queue() @@ -132,7 +130,7 @@ def run_configuration(config_filename, config, arguments): repo_queue.put((repository_path, retry_num + 1),) tuple( # Consume the generator so as to trigger logging. log_error_records( - '{}: Error running actions for repository'.format(repository_path), + f'{repository_path}: Error running actions for repository', error, levelno=logging.WARNING, log_command_error_output=True, @@ -147,7 +145,7 @@ def run_configuration(config_filename, config, arguments): return yield from log_error_records( - '{}: Error running actions for repository'.format(repository_path), error + f'{repository_path}: Error running actions for repository', error ) encountered_error = error error_repository = repository_path @@ -169,7 +167,7 @@ def run_configuration(config_filename, config, arguments): return encountered_error = error - yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error) + yield from log_error_records(f'{repository_path}: Error pinging monitor', error) if not encountered_error: try: @@ -196,7 +194,7 @@ def run_configuration(config_filename, config, arguments): return encountered_error = error - yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error) + yield from log_error_records(f'{config_filename}: Error pinging monitor', error) if encountered_error and using_primary_action: try: @@ -231,9 +229,7 @@ def run_configuration(config_filename, config, arguments): if command.considered_soft_failure(config_filename, error): return - yield from log_error_records( - '{}: Error running on-error hook'.format(config_filename), error - ) + yield from log_error_records(f'{config_filename}: Error running on-error hook', error) def run_actions( @@ -472,9 +468,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): dict( levelno=logging.WARNING, levelname='WARNING', - msg='{}: Insufficient permissions to read configuration file'.format( - config_filename - ), + msg=f'{config_filename}: Insufficient permissions to read configuration file', ) ), ] @@ -486,7 +480,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): dict( levelno=logging.CRITICAL, levelname='CRITICAL', - msg='{}: Error parsing configuration file'.format(config_filename), + msg=f'{config_filename}: Error parsing configuration file', ) ), logging.makeLogRecord( @@ -587,9 +581,7 @@ def collect_configuration_run_summary_logs(configs, arguments): if not configs: yield from log_error_records( - '{}: No valid configuration files found'.format( - ' '.join(arguments['global'].config_paths) - ) + r"{' '.join(arguments['global'].config_paths)}: No valid configuration files found", ) return @@ -615,21 +607,21 @@ def collect_configuration_run_summary_logs(configs, arguments): error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord)) if error_logs: - yield from log_error_records('{}: An error occurred'.format(config_filename)) + yield from log_error_records(f'{config_filename}: An error occurred') yield from error_logs else: yield logging.makeLogRecord( dict( levelno=logging.INFO, levelname='INFO', - msg='{}: Successfully ran configuration file'.format(config_filename), + msg=f'{config_filename}: Successfully ran configuration file', ) ) if results: json_results.extend(results) if 'umount' in arguments: - logger.info('Unmounting mount point {}'.format(arguments['umount'].mount_point)) + logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") try: borg_umount.unmount_archive( mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs), @@ -677,7 +669,7 @@ def main(): # pragma: no cover if error.code == 0: raise error configure_logging(logging.CRITICAL) - logger.critical('Error parsing arguments: {}'.format(' '.join(sys.argv))) + logger.critical(f"Error parsing arguments: {' '.join(sys.argv)}") exit_with_help_link() global_arguments = arguments['global'] @@ -710,7 +702,7 @@ def main(): # pragma: no cover ) except (FileNotFoundError, PermissionError) as error: configure_logging(logging.CRITICAL) - logger.critical('Error configuring logging: {}'.format(error)) + logger.critical(f'Error configuring logging: {error}') exit_with_help_link() logger.debug('Ensuring legacy configuration is upgraded') diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 0ff1f3e5..1fc976bc 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -34,7 +34,7 @@ def bash_completion(): ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - ' then cat << EOF\n%s\nEOF' % UPGRADE_MESSAGE, + f' then cat << EOF\n{UPGRADE_MESSAGE}\nEOF', ' fi', '}', 'complete_borgmatic() {', @@ -48,7 +48,7 @@ def bash_completion(): for action, subparser in subparsers.choices.items() ) + ( - ' COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))' + ' COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))' # noqa: FS003 % (actions, global_flags), ' (check_version &)', '}', diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index 093d4e37..64a89486 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -28,9 +28,7 @@ def parse_arguments(*arguments): '--source-config', dest='source_config_filename', default=DEFAULT_SOURCE_CONFIG_FILENAME, - help='Source INI-style configuration filename. Default: {}'.format( - DEFAULT_SOURCE_CONFIG_FILENAME - ), + help=f'Source INI-style configuration filename. Default: {DEFAULT_SOURCE_CONFIG_FILENAME}', ) parser.add_argument( '-e', @@ -46,9 +44,7 @@ def parse_arguments(*arguments): '--destination-config', dest='destination_config_filename', default=DEFAULT_DESTINATION_CONFIG_FILENAME, - help='Destination YAML configuration filename. Default: {}'.format( - DEFAULT_DESTINATION_CONFIG_FILENAME - ), + help=f'Destination YAML configuration filename. Default: {DEFAULT_DESTINATION_CONFIG_FILENAME}', ) return parser.parse_args(arguments) @@ -59,19 +55,15 @@ TEXT_WRAP_CHARACTERS = 80 def display_result(args): # pragma: no cover result_lines = textwrap.wrap( - 'Your borgmatic configuration has been upgraded. Please review the result in {}.'.format( - args.destination_config_filename - ), + f'Your borgmatic configuration has been upgraded. Please review the result in {args.destination_config_filename}.', TEXT_WRAP_CHARACTERS, ) + excludes_phrase = ( + f' and {args.source_excludes_filename}' if args.source_excludes_filename else '' + ) delete_lines = textwrap.wrap( - 'Once you are satisfied, you can safely delete {}{}.'.format( - args.source_config_filename, - ' and {}'.format(args.source_excludes_filename) - if args.source_excludes_filename - else '', - ), + f'Once you are satisfied, you can safely delete {args.source_config_filename}{excludes_phrase}.', TEXT_WRAP_CHARACTERS, ) diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py index 13a5cbaa..78c32f04 100644 --- a/borgmatic/commands/generate_config.py +++ b/borgmatic/commands/generate_config.py @@ -23,9 +23,7 @@ def parse_arguments(*arguments): '--destination', dest='destination_filename', default=DEFAULT_DESTINATION_CONFIG_FILENAME, - help='Destination YAML configuration file, default: {}'.format( - DEFAULT_DESTINATION_CONFIG_FILENAME - ), + help=f'Destination YAML configuration file, default: {DEFAULT_DESTINATION_CONFIG_FILENAME}', ) parser.add_argument( '--overwrite', @@ -48,17 +46,13 @@ def main(): # pragma: no cover overwrite=args.overwrite, ) - print('Generated a sample configuration file at {}.'.format(args.destination_filename)) + print(f'Generated a sample configuration file at {args.destination_filename}.') print() if args.source_filename: - print( - 'Merged in the contents of configuration file at {}.'.format(args.source_filename) - ) + print(f'Merged in the contents of configuration file at {args.source_filename}.') print('To review the changes made, run:') print() - print( - ' diff --unified {} {}'.format(args.source_filename, args.destination_filename) - ) + print(f' diff --unified {args.source_filename} {args.destination_filename}') print() print('This includes all available configuration options with example values. The few') print('required options are indicated. Please edit the file to suit your needs.') diff --git a/borgmatic/commands/validate_config.py b/borgmatic/commands/validate_config.py index 00ea9f45..44c0082a 100644 --- a/borgmatic/commands/validate_config.py +++ b/borgmatic/commands/validate_config.py @@ -21,9 +21,7 @@ def parse_arguments(*arguments): nargs='+', dest='config_paths', default=config_paths, - help='Configuration filenames or directories, defaults to: {}'.format( - ' '.join(config_paths) - ), + help=f'Configuration filenames or directories, defaults to: {config_paths}', ) return parser.parse_args(arguments) @@ -44,13 +42,11 @@ def main(): # pragma: no cover try: validate.parse_configuration(config_filename, validate.schema_filename()) except (ValueError, OSError, validate.Validation_error) as error: - logging.critical('{}: Error parsing configuration file'.format(config_filename)) + logging.critical(f'{config_filename}: Error parsing configuration file') logging.critical(error) found_issues = True if found_issues: sys.exit(1) else: - logger.info( - 'All given configuration files are valid: {}'.format(', '.join(config_filenames)) - ) + logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}") diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py index a13472ee..bd38fee2 100644 --- a/borgmatic/config/collect.py +++ b/borgmatic/config/collect.py @@ -16,8 +16,8 @@ def get_default_config_paths(expand_home=True): return [ '/etc/borgmatic/config.yaml', '/etc/borgmatic.d', - '%s/borgmatic/config.yaml' % user_config_directory, - '%s/borgmatic.d' % user_config_directory, + os.path.join(user_config_directory, 'borgmatic/config.yaml'), + os.path.join(user_config_directory, 'borgmatic.d'), ] diff --git a/borgmatic/config/environment.py b/borgmatic/config/environment.py index 3a58566f..a2857bbf 100644 --- a/borgmatic/config/environment.py +++ b/borgmatic/config/environment.py @@ -14,11 +14,14 @@ def _resolve_string(matcher): if matcher.group('escape') is not None: # in case of escaped envvar, unescape it return matcher.group('variable') + # resolve the env var name, default = matcher.group('name'), matcher.group('default') out = os.getenv(name, default=default) + if out is None: - raise ValueError('Cannot find variable ${name} in environment'.format(name=name)) + raise ValueError(f'Cannot find variable {name} in environment') + return out diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index e864a3c0..d486f23c 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -48,7 +48,7 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): config, schema, indent=indent, skip_first=parent_is_sequence ) else: - raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema)) + raise ValueError(f'Schema at level {level} is unsupported: {schema}') return config @@ -84,7 +84,7 @@ def _comment_out_optional_configuration(rendered_config): for line in rendered_config.split('\n'): # Upon encountering an optional configuration option, comment out lines until the next blank # line. - if line.strip().startswith('# {}'.format(COMMENTED_OUT_SENTINEL)): + if line.strip().startswith(f'# {COMMENTED_OUT_SENTINEL}'): optional = True continue @@ -117,9 +117,7 @@ def write_configuration(config_filename, rendered_config, mode=0o600, overwrite= ''' if not overwrite and os.path.exists(config_filename): raise FileExistsError( - '{} already exists. Aborting. Use --overwrite to replace the file.'.format( - config_filename - ) + f'{config_filename} already exists. Aborting. Use --overwrite to replace the file.' ) try: @@ -218,7 +216,7 @@ def remove_commented_out_sentinel(config, field_name): except KeyError: return - if last_comment_value == '# {}\n'.format(COMMENTED_OUT_SENTINEL): + if last_comment_value == f'# {COMMENTED_OUT_SENTINEL}\n': config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX].pop() diff --git a/borgmatic/config/legacy.py b/borgmatic/config/legacy.py index 91352784..ec1e50a1 100644 --- a/borgmatic/config/legacy.py +++ b/borgmatic/config/legacy.py @@ -70,13 +70,11 @@ def validate_configuration_format(parser, config_format): section_format.name for section_format in config_format ) if unknown_section_names: - raise ValueError( - 'Unknown config sections found: {}'.format(', '.join(unknown_section_names)) - ) + raise ValueError(f"Unknown config sections found: {', '.join(unknown_section_names)}") missing_section_names = set(required_section_names) - section_names if missing_section_names: - raise ValueError('Missing config sections: {}'.format(', '.join(missing_section_names))) + raise ValueError(f"Missing config sections: {', '.join(missing_section_names)}") for section_format in config_format: if section_format.name not in section_names: @@ -91,9 +89,7 @@ def validate_configuration_format(parser, config_format): if unexpected_option_names: raise ValueError( - 'Unexpected options found in config section {}: {}'.format( - section_format.name, ', '.join(sorted(unexpected_option_names)) - ) + f"Unexpected options found in config section {section_format.name}: {', '.join(sorted(unexpected_option_names))}", ) missing_option_names = tuple( @@ -105,9 +101,7 @@ def validate_configuration_format(parser, config_format): if missing_option_names: raise ValueError( - 'Required options missing from config section {}: {}'.format( - section_format.name, ', '.join(missing_option_names) - ) + f"Required options missing from config section {section_format.name}: {', '.join(missing_option_names)}", ) @@ -137,7 +131,7 @@ def parse_configuration(config_filename, config_format): ''' parser = RawConfigParser() if not parser.read(config_filename): - raise ValueError('Configuration file cannot be opened: {}'.format(config_filename)) + raise ValueError(f'Configuration file cannot be opened: {config_filename}') validate_configuration_format(parser, config_format) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 5828380e..10da19ce 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -20,9 +20,9 @@ def format_json_error_path_element(path_element): Given a path element into a JSON data structure, format it for display as a string. ''' if isinstance(path_element, int): - return str('[{}]'.format(path_element)) + return str(f'[{path_element}]') - return str('.{}'.format(path_element)) + return str(f'.{path_element}') def format_json_error(error): @@ -30,10 +30,10 @@ def format_json_error(error): Given an instance of jsonschema.exceptions.ValidationError, format it for display as a string. ''' if not error.path: - return 'At the top level: {}'.format(error.message) + return f'At the top level: {error.message}' formatted_path = ''.join(format_json_error_path_element(element) for element in error.path) - return "At '{}': {}".format(formatted_path.lstrip('.'), error.message) + return f"At '{formatted_path.lstrip('.')}': {error.message}" class Validation_error(ValueError): @@ -54,9 +54,10 @@ class Validation_error(ValueError): ''' Render a validation error as a user-facing string. ''' - return 'An error occurred while parsing a configuration file at {}:\n'.format( - self.config_filename - ) + '\n'.join(error for error in self.errors) + return ( + f'An error occurred while parsing a configuration file at {self.config_filename}:\n' + + '\n'.join(error for error in self.errors) + ) def apply_logical_validation(config_filename, parsed_configuration): @@ -72,9 +73,7 @@ def apply_logical_validation(config_filename, parsed_configuration): raise Validation_error( config_filename, ( - 'Unknown repository in the "consistency" section\'s "check_repositories": {}'.format( - repository - ), + f'Unknown repository in the "consistency" section\'s "check_repositories": {repository}', ), ) @@ -165,9 +164,9 @@ def guard_configuration_contains_repository(repository, configurations): ) if count == 0: - raise ValueError('Repository {} not found in configuration files'.format(repository)) + raise ValueError(f'Repository {repository} not found in configuration files') if count > 1: - raise ValueError('Repository {} found in multiple configuration files'.format(repository)) + raise ValueError(f'Repository {repository} found in multiple configuration files') def guard_single_repository_selected(repository, configurations): diff --git a/borgmatic/execute.py b/borgmatic/execute.py index a01e1a13..53d1a098 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -154,8 +154,8 @@ def log_command(full_command, input_file=None, output_file=None): ''' logger.debug( ' '.join(full_command) - + (' < {}'.format(getattr(input_file, 'name', '')) if input_file else '') - + (' > {}'.format(getattr(output_file, 'name', '')) if output_file else '') + + (f" < {getattr(input_file, 'name', '')}" if input_file else '') + + (f" > {getattr(output_file, 'name', '')}" if output_file else '') ) @@ -235,12 +235,12 @@ def execute_command_and_capture_output( env=environment, cwd=working_directory, ) - logger.warning('Command output: {}'.format(output)) + logger.warning(f'Command output: {output}') except subprocess.CalledProcessError as error: if exit_code_indicates_error(command, error.returncode): raise output = error.output - logger.warning('Command output: {}'.format(output)) + logger.warning(f'Command output: {output}') return output.decode() if output is not None else None diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index 756f8779..05f7d2f7 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -16,7 +16,7 @@ def interpolate_context(config_filename, hook_description, command, context): names/values, interpolate the values by "{name}" into the command and return the result. ''' for name, value in context.items(): - command = command.replace('{%s}' % name, str(value)) + command = command.replace(f'{{{name}}}', str(value)) for unsupported_variable in re.findall(r'{\w+}', command): logger.warning( @@ -38,7 +38,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte Raise subprocesses.CalledProcessError if an error occurs in a hook. ''' if not commands: - logger.debug('{}: No commands to run for {} hook'.format(config_filename, description)) + logger.debug(f'{config_filename}: No commands to run for {description} hook') return dry_run_label = ' (dry run; not actually running hooks)' if dry_run else '' @@ -49,19 +49,15 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte ] if len(commands) == 1: - logger.info( - '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label) - ) + logger.info(f'{config_filename}: Running command for {description} hook{dry_run_label}') else: logger.info( - '{}: Running {} commands for {} hook{}'.format( - config_filename, len(commands), description, dry_run_label - ) + f'{config_filename}: Running {len(commands)} commands for {description} hook{dry_run_label}', ) if umask: parsed_umask = int(str(umask), 8) - logger.debug('{}: Set hook umask to {}'.format(config_filename, oct(parsed_umask))) + logger.debug(f'{config_filename}: Set hook umask to {oct(parsed_umask)}') original_umask = os.umask(parsed_umask) else: original_umask = None @@ -93,9 +89,7 @@ def considered_soft_failure(config_filename, error): if exit_code == SOFT_FAIL_EXIT_CODE: logger.info( - '{}: Command hook exited with soft failure exit code ({}); skipping remaining actions'.format( - config_filename, SOFT_FAIL_EXIT_CODE - ) + f'{config_filename}: Command hook exited with soft failure exit code ({SOFT_FAIL_EXIT_CODE}); skipping remaining actions', ) return True diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index cd0ffa5c..05ada575 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -34,17 +34,15 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' - formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state]) + formatted_state = f'/{MONITOR_STATE_TO_CRONHUB[state]}/' ping_url = ( hook_config['ping_url'] .replace('/start/', formatted_state) .replace('/ping/', formatted_state) ) - logger.info( - '{}: Pinging Cronhub {}{}'.format(config_filename, state.name.lower(), dry_run_label) - ) - logger.debug('{}: Using Cronhub ping URL {}'.format(config_filename, ping_url)) + logger.info(f'{config_filename}: Pinging Cronhub {state.name.lower()}{dry_run_label}') + logger.debug(f'{config_filename}: Using Cronhub ping URL {ping_url}') if not dry_run: logging.getLogger('urllib3').setLevel(logging.ERROR) diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index 633b4c3c..d669c09d 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -34,12 +34,10 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' - ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state]) + ping_url = f"{hook_config['ping_url']}/{MONITOR_STATE_TO_CRONITOR[state]}" - logger.info( - '{}: Pinging Cronitor {}{}'.format(config_filename, state.name.lower(), dry_run_label) - ) - logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url)) + logger.info(f'{config_filename}: Pinging Cronitor {state.name.lower()}{dry_run_label}') + logger.debug(f'{config_filename}: Using Cronitor ping URL {ping_url}') if not dry_run: logging.getLogger('urllib3').setLevel(logging.ERROR) diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index 88a99eb1..fa7bd9b3 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -43,9 +43,9 @@ def call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs): try: module = HOOK_NAME_TO_MODULE[hook_name] except KeyError: - raise ValueError('Unknown hook name: {}'.format(hook_name)) + raise ValueError(f'Unknown hook name: {hook_name}') - logger.debug('{}: Calling {} hook function {}'.format(log_prefix, hook_name, function_name)) + logger.debug(f'{log_prefix}: Calling {hook_name} hook function {function_name}') return getattr(module, function_name)(config, log_prefix, *args, **kwargs) diff --git a/borgmatic/hooks/dump.py b/borgmatic/hooks/dump.py index 43686d36..015ed696 100644 --- a/borgmatic/hooks/dump.py +++ b/borgmatic/hooks/dump.py @@ -33,7 +33,7 @@ def make_database_dump_filename(dump_path, name, hostname=None): Raise ValueError if the database name is invalid. ''' if os.path.sep in name: - raise ValueError('Invalid database name {}'.format(name)) + raise ValueError(f'Invalid database name {name}') return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name) @@ -60,9 +60,7 @@ def remove_database_dumps(dump_path, database_type_name, log_prefix, dry_run): ''' dry_run_label = ' (dry run; not actually removing anything)' if dry_run else '' - logger.debug( - '{}: Removing {} database dumps{}'.format(log_prefix, database_type_name, dry_run_label) - ) + logger.debug(f'{log_prefix}: Removing {database_type_name} database dumps{dry_run_label}') expanded_path = os.path.expanduser(dump_path) @@ -78,4 +76,4 @@ def convert_glob_patterns_to_borg_patterns(patterns): Convert a sequence of shell glob patterns like "/etc/*" to the corresponding Borg archive patterns like "sh:etc/*". ''' - return ['sh:{}'.format(pattern.lstrip(os.path.sep)) for pattern in patterns] + return [f'sh:{pattern.lstrip(os.path.sep)}' for pattern in patterns] diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index 6ad8449f..4cafc49f 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -99,7 +99,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ ping_url = ( hook_config['ping_url'] if hook_config['ping_url'].startswith('http') - else 'https://hc-ping.com/{}'.format(hook_config['ping_url']) + else f"https://hc-ping.com/{hook_config['ping_url']}" ) dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' @@ -111,12 +111,10 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ healthchecks_state = MONITOR_STATE_TO_HEALTHCHECKS.get(state) if healthchecks_state: - ping_url = '{}/{}'.format(ping_url, healthchecks_state) + ping_url = f'{ping_url}/{healthchecks_state}' - logger.info( - '{}: Pinging Healthchecks {}{}'.format(config_filename, state.name.lower(), dry_run_label) - ) - logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url)) + logger.info(f'{config_filename}: Pinging Healthchecks {state.name.lower()}{dry_run_label}') + logger.debug(f'{config_filename}: Using Healthchecks ping URL {ping_url}') if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG): payload = format_buffered_logs_for_payload() diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index 8c3cab74..be5f656b 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -27,7 +27,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): ''' dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - logger.info('{}: Dumping MongoDB databases{}'.format(log_prefix, dry_run_label)) + logger.info(f'{log_prefix}: Dumping MongoDB databases{dry_run_label}') processes = [] for database in databases: @@ -38,9 +38,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dump_format = database.get('format', 'archive') logger.debug( - '{}: Dumping MongoDB database {} to {}{}'.format( - log_prefix, name, dump_filename, dry_run_label - ) + f'{log_prefix}: Dumping MongoDB database {name} to {dump_filename}{dry_run_label}', ) if dry_run: continue @@ -126,9 +124,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) restore_command = build_restore_command(extract_process, database, dump_filename) - logger.debug( - '{}: Restoring MongoDB database {}{}'.format(log_prefix, database['name'], dry_run_label) - ) + logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}") if dry_run: return diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index e53b8967..0bf97745 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -119,7 +119,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label)) + logger.info(f'{log_prefix}: Dumping MySQL databases{dry_run_label}') for database in databases: dump_path = make_dump_path(location_config) @@ -209,9 +209,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None - logger.debug( - '{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label) - ) + logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}") if dry_run: return diff --git a/borgmatic/hooks/pagerduty.py b/borgmatic/hooks/pagerduty.py index fbb67fbf..561b1e24 100644 --- a/borgmatic/hooks/pagerduty.py +++ b/borgmatic/hooks/pagerduty.py @@ -29,14 +29,12 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ ''' if state != monitor.State.FAIL: logger.debug( - '{}: Ignoring unsupported monitoring {} in PagerDuty hook'.format( - config_filename, state.name.lower() - ) + f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in PagerDuty hook', ) return dry_run_label = ' (dry run; not actually sending)' if dry_run else '' - logger.info('{}: Sending failure event to PagerDuty {}'.format(config_filename, dry_run_label)) + logger.info(f'{config_filename}: Sending failure event to PagerDuty {dry_run_label}') if dry_run: return @@ -50,7 +48,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ 'routing_key': hook_config['integration_key'], 'event_action': 'trigger', 'payload': { - 'summary': 'backup failed on {}'.format(hostname), + 'summary': f'backup failed on {hostname}', 'severity': 'error', 'source': hostname, 'timestamp': local_timestamp, @@ -65,7 +63,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ }, } ) - logger.debug('{}: Using PagerDuty payload: {}'.format(config_filename, payload)) + logger.debug(f'{config_filename}: Using PagerDuty payload: {payload}') logging.getLogger('urllib3').setLevel(logging.ERROR) try: diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 3d3676fe..d4799f5f 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -93,7 +93,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info('{}: Dumping PostgreSQL databases{}'.format(log_prefix, dry_run_label)) + logger.info(f'{log_prefix}: Dumping PostgreSQL databases{dry_run_label}') for database in databases: extra_environment = make_extra_environment(database) @@ -228,9 +228,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) extra_environment = make_extra_environment(database) - logger.debug( - '{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label) - ) + logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") if dry_run: return diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py index 9e7ecf37..d9f105d8 100644 --- a/borgmatic/hooks/sqlite.py +++ b/borgmatic/hooks/sqlite.py @@ -26,7 +26,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info('{}: Dumping SQLite databases{}'.format(log_prefix, dry_run_label)) + logger.info(f'{log_prefix}: Dumping SQLite databases{dry_run_label}') for database in databases: database_path = database['path'] diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 0916bfa9..e098bf96 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -108,7 +108,7 @@ def color_text(color, message): if not color: return message - return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL) + return f'{color}{message}{colorama.Style.RESET_ALL}' def add_logging_level(level_name, level_number): diff --git a/test_requirements.txt b/test_requirements.txt index 9cae8fb4..d34bc623 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -6,6 +6,8 @@ colorama==0.4.4 coverage==5.3 flake8==4.0.1 flake8-quotes==3.3.2 +flake8-use-fstring==1.4 +flake8-variables-names==0.0.5 flexmock==0.10.4 isort==5.9.1 mccabe==0.6.1 diff --git a/tests/end-to-end/test_borgmatic.py b/tests/end-to-end/test_borgmatic.py index c2d10291..de38bbd0 100644 --- a/tests/end-to-end/test_borgmatic.py +++ b/tests/end-to-end/test_borgmatic.py @@ -12,17 +12,15 @@ def generate_configuration(config_path, repository_path): to work for testing (including injecting the given repository path and tacking on an encryption passphrase). ''' - subprocess.check_call( - 'generate-borgmatic-config --destination {}'.format(config_path).split(' ') - ) + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) config = ( open(config_path) .read() .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) - .replace('- ssh://user@backupserver/./{fqdn}', '') + .replace('- ssh://user@backupserver/./{fqdn}', '') # noqa: FS003 .replace('- /var/local/backups/local.borg', '') .replace('- /home/user/path with spaces', '') - .replace('- /home', '- {}'.format(config_path)) + .replace('- /home', f'- {config_path}') .replace('- /etc', '') .replace('- /var/log/syslog*', '') + 'storage:\n encryption_passphrase: "test"' @@ -47,13 +45,13 @@ def test_borgmatic_command(): generate_configuration(config_path, repository_path) subprocess.check_call( - 'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ') + f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ') ) # Run borgmatic to generate a backup archive, and then list it to make sure it exists. - subprocess.check_call('borgmatic --config {}'.format(config_path).split(' ')) + subprocess.check_call(f'borgmatic --config {config_path}'.split(' ')) output = subprocess.check_output( - 'borgmatic --config {} list --json'.format(config_path).split(' ') + f'borgmatic --config {config_path} list --json'.split(' ') ).decode(sys.stdout.encoding) parsed_output = json.loads(output) @@ -64,16 +62,14 @@ def test_borgmatic_command(): # Extract the created archive into the current (temporary) directory, and confirm that the # extracted file looks right. output = subprocess.check_output( - 'borgmatic --config {} extract --archive {}'.format(config_path, archive_name).split( - ' ' - ) + f'borgmatic --config {config_path} extract --archive {archive_name}'.split(' '), ).decode(sys.stdout.encoding) extracted_config_path = os.path.join(extract_path, config_path) assert open(extracted_config_path).read() == open(config_path).read() # Exercise the info action. output = subprocess.check_output( - 'borgmatic --config {} info --json'.format(config_path).split(' ') + f'borgmatic --config {config_path} info --json'.split(' '), ).decode(sys.stdout.encoding) parsed_output = json.loads(output) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 8849b3c8..30aea4a8 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -189,7 +189,7 @@ def test_database_dump_with_error_causes_borgmatic_to_exit(): '-v', '2', '--override', - "hooks.postgresql_databases=[{'name': 'nope'}]", + "hooks.postgresql_databases=[{'name': 'nope'}]", # noqa: FS003 ] ) finally: diff --git a/tests/end-to-end/test_override.py b/tests/end-to-end/test_override.py index 0a42018d..e86186d9 100644 --- a/tests/end-to-end/test_override.py +++ b/tests/end-to-end/test_override.py @@ -10,17 +10,15 @@ def generate_configuration(config_path, repository_path): to work for testing (including injecting the given repository path and tacking on an encryption passphrase). ''' - subprocess.check_call( - 'generate-borgmatic-config --destination {}'.format(config_path).split(' ') - ) + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) config = ( open(config_path) .read() .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) - .replace('- ssh://user@backupserver/./{fqdn}', '') + .replace('- ssh://user@backupserver/./{fqdn}', '') # noqa: FS003 .replace('- /var/local/backups/local.borg', '') .replace('- /home/user/path with spaces', '') - .replace('- /home', '- {}'.format(config_path)) + .replace('- /home', f'- {config_path}') .replace('- /etc', '') .replace('- /var/log/syslog*', '') + 'storage:\n encryption_passphrase: "test"' diff --git a/tests/end-to-end/test_validate_config.py b/tests/end-to-end/test_validate_config.py index 5de83a39..d41464e6 100644 --- a/tests/end-to-end/test_validate_config.py +++ b/tests/end-to-end/test_validate_config.py @@ -7,12 +7,8 @@ def test_validate_config_command_with_valid_configuration_succeeds(): with tempfile.TemporaryDirectory() as temporary_directory: config_path = os.path.join(temporary_directory, 'test.yaml') - subprocess.check_call( - 'generate-borgmatic-config --destination {}'.format(config_path).split(' ') - ) - exit_code = subprocess.call( - 'validate-borgmatic-config --config {}'.format(config_path).split(' ') - ) + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' ')) assert exit_code == 0 @@ -21,16 +17,12 @@ def test_validate_config_command_with_invalid_configuration_fails(): with tempfile.TemporaryDirectory() as temporary_directory: config_path = os.path.join(temporary_directory, 'test.yaml') - subprocess.check_call( - 'generate-borgmatic-config --destination {}'.format(config_path).split(' ') - ) + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) config = open(config_path).read().replace('keep_daily: 7', 'keep_daily: "7"') config_file = open(config_path, 'w') config_file.write(config) config_file.close() - exit_code = subprocess.call( - 'validate-borgmatic-config --config {}'.format(config_path).split(' ') - ) + exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' ')) assert exit_code == 1 diff --git a/tests/integration/config/test_legacy.py b/tests/integration/config/test_legacy.py index 870da886..c73e7eec 100644 --- a/tests/integration/config/test_legacy.py +++ b/tests/integration/config/test_legacy.py @@ -7,7 +7,7 @@ from borgmatic.config import legacy as module def test_parse_section_options_with_punctuation_should_return_section_options(): parser = module.RawConfigParser() - parser.read_file(StringIO('[section]\nfoo: {}\n'.format(string.punctuation))) + parser.read_file(StringIO(f'[section]\nfoo: {string.punctuation}\n')) section_format = module.Section_format( 'section', (module.Config_option('foo', str, required=True),) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 69a3ede2..5fb51f3c 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -449,7 +449,7 @@ def test_collect_special_file_paths_excludes_non_special_files(): ) == ('/foo', '/baz') -DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' +DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003 REPO_ARCHIVE_WITH_PATHS = (f'repo::{DEFAULT_ARCHIVE_NAME}', 'foo', 'bar') @@ -2193,7 +2193,7 @@ def test_create_archive_with_source_directories_glob_expands(): ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'), + ('borg', 'create', f'repo::{DEFAULT_ARCHIVE_NAME}', 'foo', 'food'), output_log_level=logging.INFO, output_file=None, borg_local_path='borg', @@ -2236,7 +2236,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'), + ('borg', 'create', f'repo::{DEFAULT_ARCHIVE_NAME}', 'foo*'), output_log_level=logging.INFO, output_file=None, borg_local_path='borg', @@ -2279,7 +2279,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'), + ('borg', 'create', f'repo::{DEFAULT_ARCHIVE_NAME}', 'foo', 'food'), output_log_level=logging.INFO, output_file=None, borg_local_path='borg', @@ -2345,7 +2345,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - repository_archive_pattern = 'repo::Documents_{hostname}-{now}' + repository_archive_pattern = 'repo::Documents_{hostname}-{now}' # noqa: FS003 flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('map_directories_to_devices').and_return({}) @@ -2380,7 +2380,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, + storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', ) @@ -2388,7 +2388,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): def test_create_archive_with_repository_accepts_borg_placeholders(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - repository_archive_pattern = '{fqdn}::Documents_{hostname}-{now}' + repository_archive_pattern = '{fqdn}::Documents_{hostname}-{now}' # noqa: FS003 flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('map_directories_to_devices').and_return({}) @@ -2417,13 +2417,13 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): module.create_archive( dry_run=False, - repository='{fqdn}', + repository='{fqdn}', # noqa: FS003 location_config={ 'source_directories': ['foo', 'bar'], - 'repositories': ['{fqdn}'], + 'repositories': ['{fqdn}'], # noqa: FS003 'exclude_patterns': None, }, - storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, + storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index ed4101e2..dd240dcb 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -27,27 +27,39 @@ def test_make_prune_flags_returns_flags_from_config_plus_default_prefix_glob(): result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') - assert tuple(result) == BASE_PRUNE_FLAGS + (('--match-archives', 'sh:{hostname}-*'),) + assert tuple(result) == BASE_PRUNE_FLAGS + ( + ('--match-archives', 'sh:{hostname}-*'), # noqa: FS003 + ) def test_make_prune_flags_accepts_prefix_with_placeholders(): - retention_config = OrderedDict((('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}'))) + retention_config = OrderedDict( + (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 + ) flexmock(module.feature).should_receive('available').and_return(True) result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') - expected = (('--keep-daily', '1'), ('--match-archives', 'sh:Documents_{hostname}-{now}*')) + expected = ( + ('--keep-daily', '1'), + ('--match-archives', 'sh:Documents_{hostname}-{now}*'), # noqa: FS003 + ) assert tuple(result) == expected def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives(): - retention_config = OrderedDict((('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}'))) + retention_config = OrderedDict( + (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 + ) flexmock(module.feature).should_receive('available').and_return(False) result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') - expected = (('--keep-daily', '1'), ('--glob-archives', 'Documents_{hostname}-{now}*')) + expected = ( + ('--keep-daily', '1'), + ('--glob-archives', 'Documents_{hostname}-{now}*'), # noqa: FS003 + ) assert tuple(result) == expected diff --git a/tests/unit/config/test_environment.py b/tests/unit/config/test_environment.py index b7b56dd8..3e342fa0 100644 --- a/tests/unit/config/test_environment.py +++ b/tests/unit/config/test_environment.py @@ -12,7 +12,7 @@ def test_env(monkeypatch): def test_env_braces(monkeypatch): monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo') - config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} + config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} # noqa: FS003 module.resolve_env_variables(config) assert config == {'key': 'Hello foo'} @@ -20,7 +20,7 @@ def test_env_braces(monkeypatch): def test_env_multi(monkeypatch): monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo') monkeypatch.setenv('MY_CUSTOM_VALUE2', 'bar') - config = {'key': 'Hello ${MY_CUSTOM_VALUE}${MY_CUSTOM_VALUE2}'} + config = {'key': 'Hello ${MY_CUSTOM_VALUE}${MY_CUSTOM_VALUE2}'} # noqa: FS003 module.resolve_env_variables(config) assert config == {'key': 'Hello foobar'} @@ -28,21 +28,21 @@ def test_env_multi(monkeypatch): def test_env_escape(monkeypatch): monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo') monkeypatch.setenv('MY_CUSTOM_VALUE2', 'bar') - config = {'key': r'Hello ${MY_CUSTOM_VALUE} \${MY_CUSTOM_VALUE}'} + config = {'key': r'Hello ${MY_CUSTOM_VALUE} \${MY_CUSTOM_VALUE}'} # noqa: FS003 module.resolve_env_variables(config) - assert config == {'key': r'Hello foo ${MY_CUSTOM_VALUE}'} + assert config == {'key': r'Hello foo ${MY_CUSTOM_VALUE}'} # noqa: FS003 def test_env_default_value(monkeypatch): monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False) - config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'} + config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'} # noqa: FS003 module.resolve_env_variables(config) assert config == {'key': 'Hello bar'} def test_env_unknown(monkeypatch): monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False) - config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} + config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} # noqa: FS003 with pytest.raises(ValueError): module.resolve_env_variables(config) @@ -55,20 +55,20 @@ def test_env_full(monkeypatch): 'dict': { 'key': 'value', 'anotherdict': { - 'key': 'My ${MY_CUSTOM_VALUE} here', - 'other': '${MY_CUSTOM_VALUE}', - 'escaped': r'\${MY_CUSTOM_VALUE}', + 'key': 'My ${MY_CUSTOM_VALUE} here', # noqa: FS003 + 'other': '${MY_CUSTOM_VALUE}', # noqa: FS003 + 'escaped': r'\${MY_CUSTOM_VALUE}', # noqa: FS003 'list': [ - '/home/${MY_CUSTOM_VALUE}/.local', + '/home/${MY_CUSTOM_VALUE}/.local', # noqa: FS003 '/var/log/', - '/home/${MY_CUSTOM_VALUE2:-bar}/.config', + '/home/${MY_CUSTOM_VALUE2:-bar}/.config', # noqa: FS003 ], }, }, 'list': [ - '/home/${MY_CUSTOM_VALUE}/.local', + '/home/${MY_CUSTOM_VALUE}/.local', # noqa: FS003 '/var/log/', - '/home/${MY_CUSTOM_VALUE2-bar}/.config', + '/home/${MY_CUSTOM_VALUE2-bar}/.config', # noqa: FS003 ], } module.resolve_env_variables(config) @@ -79,7 +79,7 @@ def test_env_full(monkeypatch): 'anotherdict': { 'key': 'My foo here', 'other': 'foo', - 'escaped': '${MY_CUSTOM_VALUE}', + 'escaped': '${MY_CUSTOM_VALUE}', # noqa: FS003 'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'], }, }, diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 6a9f4a4e..28527226 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -13,7 +13,7 @@ def test_format_json_error_path_element_formats_property(): def test_format_json_error_formats_error_including_path(): - flexmock(module).format_json_error_path_element = lambda element: '.{}'.format(element) + flexmock(module).format_json_error_path_element = lambda element: f'.{element}' error = flexmock(message='oops', path=['foo', 'bar']) assert module.format_json_error(error) == "At 'foo.bar': oops" @@ -66,9 +66,9 @@ def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_pref module.apply_logical_validation( 'config.yaml', { - 'storage': {'archive_name_format': '{hostname}-{now}'}, - 'retention': {'prefix': '{hostname}-'}, - 'consistency': {'prefix': '{hostname}-'}, + 'storage': {'archive_name_format': '{hostname}-{now}'}, # noqa: FS003 + 'retention': {'prefix': '{hostname}-'}, # noqa: FS003 + 'consistency': {'prefix': '{hostname}-'}, # noqa: FS003 }, ) diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index 3d1686d0..3a657eb8 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -11,27 +11,20 @@ def test_interpolate_context_passes_through_command_without_variable(): def test_interpolate_context_passes_through_command_with_unknown_variable(): - assert ( - module.interpolate_context('test.yaml', 'pre-backup', 'ls {baz}', {'foo': 'bar'}) - == 'ls {baz}' - ) + command = 'ls {baz}' # noqa: FS003 + + assert module.interpolate_context('test.yaml', 'pre-backup', command, {'foo': 'bar'}) == command def test_interpolate_context_interpolates_variables(): + command = 'ls {foo}{baz} {baz}' # noqa: FS003 context = {'foo': 'bar', 'baz': 'quux'} assert ( - module.interpolate_context('test.yaml', 'pre-backup', 'ls {foo}{baz} {baz}', context) - == 'ls barquux quux' + module.interpolate_context('test.yaml', 'pre-backup', command, context) == 'ls barquux quux' ) -def test_interpolate_context_does_not_touch_unknown_variables(): - context = {'foo': 'bar', 'baz': 'quux'} - - assert module.interpolate_context('test.yaml', 'pre-backup', 'ls {wtf}', context) == 'ls {wtf}' - - def test_execute_hook_invokes_each_command(): flexmock(module).should_receive('interpolate_context').replace_with( lambda config_file, hook_description, command, context: command diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index d5779534..c975e4fd 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -206,9 +206,7 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): payload = 'data' flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload) flexmock(module.requests).should_receive('post').with_args( - 'https://hc-ping.com/{}'.format(hook_config['ping_url']), - data=payload.encode('utf-8'), - verify=True, + f"https://hc-ping.com/{hook_config['ping_url']}", data=payload.encode('utf-8'), verify=True, ).and_return(flexmock(ok=True)) module.ping_monitor( diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index f61f3c70..44e427f1 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -17,7 +17,7 @@ def test_dump_databases_runs_mongodump_for_each_database(): for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_command').with_args( - ['mongodump', '--db', name, '--archive', '>', 'databases/localhost/{}'.format(name)], + ['mongodump', '--db', name, '--archive', '>', f'databases/localhost/{name}'], shell=True, run_to_completion=False, ).and_return(process).once() diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 9cb4c0ff..349c04be 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -134,7 +134,7 @@ def test_dump_databases_runs_pg_dump_for_each_database(): 'custom', name, '>', - 'databases/localhost/{}'.format(name), + f'databases/localhost/{name}', ), shell=True, extra_environment={'PGSSLMODE': 'disable'}, From e83ad9e1e4cb3dfb2209268f51a0f853825c6fc4 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 01:04:57 +0530 Subject: [PATCH 13/94] use repository["path"] instead of repository --- borgmatic/actions/borg.py | 6 ++-- borgmatic/actions/break_lock.py | 8 +++-- borgmatic/actions/check.py | 4 +-- borgmatic/actions/compact.py | 8 +++-- borgmatic/actions/export_tar.py | 8 +++-- borgmatic/actions/extract.py | 8 +++-- borgmatic/actions/info.py | 6 ++-- borgmatic/actions/list.py | 8 ++--- borgmatic/actions/mount.py | 10 +++--- borgmatic/actions/prune.py | 4 +-- borgmatic/actions/rcreate.py | 4 +-- borgmatic/actions/restore.py | 27 ++++++++++++----- borgmatic/actions/rinfo.py | 6 ++-- borgmatic/actions/rlist.py | 4 +-- borgmatic/commands/borgmatic.py | 4 +++ borgmatic/config/normalize.py | 37 ++++++++++++++--------- borgmatic/config/schema.yaml | 36 ++++++++++++++-------- borgmatic/config/validate.py | 14 +++++---- tests/integration/config/test_validate.py | 21 +++++++++---- tests/unit/actions/test_borg.py | 2 +- tests/unit/actions/test_break_lock.py | 2 +- tests/unit/actions/test_check.py | 6 ++-- tests/unit/actions/test_compact.py | 6 ++-- tests/unit/actions/test_create.py | 4 +-- tests/unit/actions/test_export_tar.py | 2 +- tests/unit/actions/test_extract.py | 2 +- tests/unit/actions/test_info.py | 2 +- tests/unit/actions/test_list.py | 2 +- tests/unit/actions/test_mount.py | 2 +- tests/unit/actions/test_prune.py | 4 +-- tests/unit/actions/test_rcreate.py | 4 +-- tests/unit/actions/test_restore.py | 16 +++++----- tests/unit/actions/test_rinfo.py | 2 +- tests/unit/actions/test_rlist.py | 2 +- tests/unit/commands/test_borgmatic.py | 36 +++++++++++----------- tests/unit/config/test_normalize.py | 10 +++--- 36 files changed, 193 insertions(+), 134 deletions(-) diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index a50dd286..bf9abd5a 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -16,9 +16,9 @@ def run_borg( if borg_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, borg_arguments.repository ): - logger.info('{}: Running arbitrary Borg command'.format(repository)) + logger.info('{}: Running arbitrary Borg command'.format(repository['path'])) archive_name = borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], borg_arguments.archive, storage, local_borg_version, @@ -26,7 +26,7 @@ def run_borg( remote_path, ) borgmatic.borg.borg.run_arbitrary_borg( - repository, + repository['path'], storage, local_borg_version, options=borg_arguments.options, diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index 65384d7a..eb6e4547 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -15,7 +15,11 @@ def run_break_lock( if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, break_lock_arguments.repository ): - logger.info(f'{repository}: Breaking repository and cache locks') + logger.info(f'{repository["path"]}: Breaking repository and cache locks') borgmatic.borg.break_lock.break_lock( - repository, storage, local_borg_version, local_path=local_path, remote_path=remote_path, + repository['path'], + storage, + local_borg_version, + local_path=local_path, + remote_path=remote_path, ) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index f3572395..ff8225ec 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -37,9 +37,9 @@ def run_check( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Running consistency checks'.format(repository)) + logger.info('{}: Running consistency checks'.format(repository['path'])) borgmatic.borg.check.check_archives( - repository, + repository['path'], location, storage, consistency, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 7a25b829..13bb28ea 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -39,10 +39,10 @@ def run_compact( **hook_context, ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): - logger.info('{}: Compacting segments{}'.format(repository, dry_run_label)) + logger.info('{}: Compacting segments{}'.format(repository['path'], dry_run_label)) borgmatic.borg.compact.compact_segments( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, local_path=local_path, @@ -52,7 +52,9 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover - logger.info('{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository)) + logger.info( + '{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository['path']) + ) borgmatic.hooks.command.execute_hook( hooks.get('after_compact'), hooks.get('umask'), diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index ae349208..7599a23a 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -23,13 +23,15 @@ def run_export_tar( repository, export_tar_arguments.repository ): logger.info( - '{}: Exporting archive {} as tar file'.format(repository, export_tar_arguments.archive) + '{}: Exporting archive {} as tar file'.format( + repository['path'], export_tar_arguments.archive + ) ) borgmatic.borg.export_tar.export_tar_archive( global_arguments.dry_run, - repository, + repository['path'], borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], export_tar_arguments.archive, storage, local_borg_version, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index a3d89a55..a36c498b 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -35,12 +35,14 @@ def run_extract( if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): - logger.info('{}: Extracting archive {}'.format(repository, extract_arguments.archive)) + logger.info( + '{}: Extracting archive {}'.format(repository['path'], extract_arguments.archive) + ) borgmatic.borg.extract.extract_archive( global_arguments.dry_run, - repository, + repository['path'], borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], extract_arguments.archive, storage, local_borg_version, diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index ab4fe426..6cd775f5 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -20,9 +20,9 @@ def run_info( repository, info_arguments.repository ): if not info_arguments.json: # pragma: nocover - logger.answer(f'{repository}: Displaying archive summary information') + logger.answer(f'{repository["path"]}: Displaying archive summary information') info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], info_arguments.archive, storage, local_borg_version, @@ -30,7 +30,7 @@ def run_info( remote_path, ) json_output = borgmatic.borg.info.display_archives_info( - repository, + repository['path'], storage, local_borg_version, info_arguments=info_arguments, diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 78efdf59..0e736f84 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -20,11 +20,11 @@ def run_list( ): if not list_arguments.json: # pragma: nocover if list_arguments.find_paths: - logger.answer(f'{repository}: Searching archives') + logger.answer(f'{repository["path"]}: Searching archives') elif not list_arguments.archive: - logger.answer(f'{repository}: Listing archives') + logger.answer(f'{repository["path"]}: Listing archives') list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], list_arguments.archive, storage, local_borg_version, @@ -32,7 +32,7 @@ def run_list( remote_path, ) json_output = borgmatic.borg.list.list_archive( - repository, + repository['path'], storage, local_borg_version, list_arguments=list_arguments, diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index e2703a57..d2c1821d 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -17,14 +17,16 @@ def run_mount( repository, mount_arguments.repository ): if mount_arguments.archive: - logger.info('{}: Mounting archive {}'.format(repository, mount_arguments.archive)) + logger.info( + '{}: Mounting archive {}'.format(repository['path'], mount_arguments.archive) + ) else: # pragma: nocover - logger.info('{}: Mounting repository'.format(repository)) + logger.info('{}: Mounting repository'.format(repository['path'])) borgmatic.borg.mount.mount_archive( - repository, + repository['path'], borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], mount_arguments.archive, storage, local_borg_version, diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index ca098ce4..6ba28a92 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -37,10 +37,10 @@ def run_prune( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Pruning archives{}'.format(repository, dry_run_label)) + logger.info('{}: Pruning archives{}'.format(repository['path'], dry_run_label)) borgmatic.borg.prune.prune_archives( global_arguments.dry_run, - repository, + repository['path'], storage, retention, local_borg_version, diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index 0052b4b6..92dcfe34 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -23,10 +23,10 @@ def run_rcreate( ): return - logger.info('{}: Creating repository'.format(repository)) + logger.info('{}: Creating repository'.format(repository['path'])) borgmatic.borg.rcreate.create_repository( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, rcreate_arguments.encryption_mode, diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 7a058092..793b8698 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -256,22 +256,35 @@ def run_restore( return logger.info( - '{}: Restoring databases from archive {}'.format(repository, restore_arguments.archive) + '{}: Restoring databases from archive {}'.format( + repository['path'], restore_arguments.archive + ) ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, ) archive_name = borgmatic.borg.rlist.resolve_archive_name( - repository, restore_arguments.archive, storage, local_borg_version, local_path, remote_path, + repository['path'], + restore_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, ) archive_database_names = collect_archive_database_names( - repository, archive_name, location, storage, local_borg_version, local_path, remote_path, + repository['path'], + archive_name, + location, + storage, + local_borg_version, + local_path, + remote_path, ) restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names) found_names = set() @@ -291,7 +304,7 @@ def run_restore( found_names.add(database_name) restore_single_database( - repository, + repository['path'], location, storage, hooks, @@ -320,7 +333,7 @@ def run_restore( database['name'] = database_name restore_single_database( - repository, + repository['path'], location, storage, hooks, @@ -336,7 +349,7 @@ def run_restore( borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 611d1bc2..5e0a41a7 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -19,9 +19,11 @@ def run_rinfo( repository, rinfo_arguments.repository ): if not rinfo_arguments.json: # pragma: nocover - logger.answer('{}: Displaying repository summary information'.format(repository)) + logger.answer( + '{}: Displaying repository summary information'.format(repository['path']) + ) json_output = borgmatic.borg.rinfo.display_repository_info( - repository, + repository['path'], storage, local_borg_version, rinfo_arguments=rinfo_arguments, diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index 72d52068..29e500c1 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -19,9 +19,9 @@ def run_rlist( repository, rlist_arguments.repository ): if not rlist_arguments.json: # pragma: nocover - logger.answer('{}: Listing repository'.format(repository)) + logger.answer('{}: Listing repository'.format(repository['path'])) json_output = borgmatic.borg.rlist.list_repository( - repository, + repository['path'], storage, local_borg_version, rlist_arguments=rlist_arguments, diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 1f8b15f4..b52bc568 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -109,6 +109,8 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository, retry_num = repo_queue.get() + if isinstance(repository, str): + repository = {'path': repository} timeout = retry_num * retry_wait if timeout: logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') @@ -263,6 +265,8 @@ def run_actions( invalid. ''' add_custom_log_levels() + if isinstance(repository, str): + repository = {'path': repository} repository_path = os.path.expanduser(repository['path']) global_arguments = arguments['global'] dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 8ce4af22..eb2ed013 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -56,14 +56,16 @@ def normalize(config_filename, config): # Upgrade remote repositories to ssh:// syntax, required in Borg 2. repositories = location.get('repositories') - if isinstance(repositories[0], str): - config['location']['repositories'] = [{'path': repository} for repository in repositories] - repositories = config['location']['repositories'] if repositories: + if isinstance(repositories[0], str): + config['location']['repositories'] = [ + {'path': repository} for repository in repositories + ] + repositories = config['location']['repositories'] config['location']['repositories'] = [] for repository_dict in repositories: - repository = repository_dict['path'] - if '~' in repository: + repository_path = repository_dict['path'] + if '~' in repository_path: logs.append( logging.makeLogRecord( dict( @@ -73,37 +75,42 @@ def normalize(config_filename, config): ) ) ) - if ':' in repository: - if repository.startswith('file://'): - updated_repository_path = os.path.abspath(repository.partition('file://')[-1]) + if ':' in repository_path: + if repository_path.startswith('file://'): + updated_repository_path = os.path.abspath( + repository_path.partition('file://')[-1] + ) config['location']['repositories'].append( { 'path': updated_repository_path, - 'label': repository_dict.get('label', None), + 'label': repository_dict.get('label', ''), } ) - elif repository.startswith('ssh://'): + elif repository_path.startswith('ssh://'): config['location']['repositories'].append( - {'path': repository, 'label': repository_dict.get('label', None),} + {'path': repository_path, 'label': repository_dict.get('label', '')} ) else: - rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" + rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( logging.makeLogRecord( dict( levelno=logging.WARNING, levelname='WARNING', - msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository}" as "{rewritten_repository}"', + msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository_path}" as "{rewritten_repository_path}"', ) ) ) config['location']['repositories'].append( - {'path': rewritten_repository, 'label': repository_dict.get('label', None),} + { + 'path': rewritten_repository_path, + 'label': repository_dict.get('label', ''), + } ) else: config['location']['repositories'].append( - {'path': repository, 'label': repository_dict.get('label', None),} + {'path': repository_path, 'label': repository_dict.get('label', '')} ) return logs diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 4235adba..14d028b7 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -36,23 +36,33 @@ properties: path: type: string description: | - Path to local or remote repository (required). - are expanded. Multiple repositories are backed up to - in sequence. Borg placeholders can be used. See the - output of "borg help placeholders" for details. See - ssh_command for SSH options like identity file or - port. If systemd service is used, then add local - repository paths in the systemd service file to the - ReadWritePaths list. - example: - - ssh://user@backupserver/./sourcehostname.borg - - ssh://user@backupserver/./{fqdn} - - /var/local/backups/local.borg + Path to local or remote repository + (required). Tildes are expanded. + Multiple repositories are backed up + to in sequence. Borg placeholders + can be used. See the output of + "borg help placeholders" for + details. See ssh_command for SSH + options like identity file or port. + If systemd service is used, then + add local repository paths in the + systemd service file to the + ReadWritePaths list. + example: ssh://user@backupserver/./{fqdn} label: type: string description: | - Optional label for the repository. + Optional label for the repository. This + can be used with the --repository option + to select a repository to backup to. + If not specified, the repository path is + used as the label. example: backupserver + example: + - path: ssh://user@backupserver/./{fqdn} + label: backupserver + - path: /mnt/backup + label: local working_directory: type: string description: | diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index b43d42df..ecacec71 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -141,12 +141,14 @@ def repositories_match(first, second): Given two repository dicts with keys 'path' (relative and/or absolute), and 'label', return whether they match. ''' - if isinstance(first, str) and isinstance(second, str): - return normalize_repository_path(first) == normalize_repository_path(second) - elif isinstance(first, dict) and isinstance(second, str): - return (second == first.get('label')) or ( - normalize_repository_path(second) == normalize_repository_path(first.get('path')) - ) + if isinstance(first, str): + first = {'path': first, 'label': first} + if isinstance(second, str): + second = {'path': second, 'label': second} + return (first.get('label') == second.get('label')) or ( + normalize_repository_path(first.get('path')) + == normalize_repository_path(second.get('path')) + ) def guard_configuration_contains_repository(repository, configurations): diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 5d948ae2..31a85cd1 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -63,7 +63,10 @@ def test_parse_configuration_transforms_file_into_mapping(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']}, + 'location': { + 'source_directories': ['/home', '/etc'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], + }, 'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60}, 'consistency': {'checks': [{'name': 'repository'}, {'name': 'archives'}]}, } @@ -89,7 +92,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): assert config == { 'location': { 'source_directories': [f'/home/{string.punctuation}'], - 'repositories': ['test.borg'], + 'repositories': [{'path': 'test.borg', 'label': ''}], } } assert logs == [] @@ -151,7 +154,10 @@ def test_parse_configuration_inlines_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']}, + 'location': { + 'source_directories': ['/home'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], + }, 'retention': {'keep_daily': 7, 'keep_hourly': 24}, } assert logs == [] @@ -185,7 +191,10 @@ def test_parse_configuration_merges_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']}, + 'location': { + 'source_directories': ['/home'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], + }, 'retention': {'keep_daily': 1, 'keep_hourly': 24}, } assert logs == [] @@ -247,7 +256,7 @@ def test_parse_configuration_applies_overrides(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': ['hostname.borg'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], 'local_path': 'borg2', } } @@ -273,7 +282,7 @@ def test_parse_configuration_applies_normalization(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': ['hostname.borg'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], 'exclude_if_present': ['.nobackup'], } } diff --git a/tests/unit/actions/test_borg.py b/tests/unit/actions/test_borg.py index 7b22c7b3..f597acbf 100644 --- a/tests/unit/actions/test_borg.py +++ b/tests/unit/actions/test_borg.py @@ -13,7 +13,7 @@ def test_run_borg_does_not_raise(): borg_arguments = flexmock(repository=flexmock(), archive=flexmock(), options=flexmock()) module.run_borg( - repository='repo', + repository={'path': 'repos'}, storage={}, local_borg_version=None, borg_arguments=borg_arguments, diff --git a/tests/unit/actions/test_break_lock.py b/tests/unit/actions/test_break_lock.py index c7db00bc..6dc2470e 100644 --- a/tests/unit/actions/test_break_lock.py +++ b/tests/unit/actions/test_break_lock.py @@ -10,7 +10,7 @@ def test_run_break_lock_does_not_raise(): break_lock_arguments = flexmock(repository=flexmock()) module.run_break_lock( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, break_lock_arguments=break_lock_arguments, diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 3e1a9c2f..4c2027eb 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -18,7 +18,7 @@ def test_run_check_calls_hooks_for_configured_repository(): module.run_check( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, consistency={}, @@ -49,7 +49,7 @@ def test_run_check_runs_with_selected_repository(): module.run_check( config_filename='test.yaml', - repository=flexmock(), + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, consistency={}, @@ -80,7 +80,7 @@ def test_run_check_bails_if_repository_does_not_match(): module.run_check( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, consistency={}, diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 4dae903e..fbd4f905 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -16,7 +16,7 @@ def test_compact_actions_calls_hooks_for_configured_repository(): module.run_compact( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, @@ -44,7 +44,7 @@ def test_compact_runs_with_selected_repository(): module.run_compact( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, @@ -72,7 +72,7 @@ def test_compact_bails_if_repository_does_not_match(): module.run_compact( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 8a9d0b4e..2b724085 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -24,7 +24,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): list( module.run_create( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={}, storage={}, hooks={}, @@ -57,7 +57,7 @@ def test_run_create_runs_with_selected_repository(): list( module.run_create( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={}, storage={}, hooks={}, diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index 41b680af..6741d427 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -19,7 +19,7 @@ def test_run_export_tar_does_not_raise(): global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_export_tar( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, export_tar_arguments=export_tar_arguments, diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index 4222b8a8..32b93b4e 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -20,7 +20,7 @@ def test_run_extract_calls_hooks(): module.run_extract( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, hooks={}, diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py index 8cde178d..a4f1d544 100644 --- a/tests/unit/actions/test_info.py +++ b/tests/unit/actions/test_info.py @@ -14,7 +14,7 @@ def test_run_info_does_not_raise(): list( module.run_info( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, info_arguments=info_arguments, diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py index f4a603de..bfdfd010 100644 --- a/tests/unit/actions/test_list.py +++ b/tests/unit/actions/test_list.py @@ -14,7 +14,7 @@ def test_run_list_does_not_raise(): list( module.run_list( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, list_arguments=list_arguments, diff --git a/tests/unit/actions/test_mount.py b/tests/unit/actions/test_mount.py index 0624a896..7eadfca1 100644 --- a/tests/unit/actions/test_mount.py +++ b/tests/unit/actions/test_mount.py @@ -17,7 +17,7 @@ def test_run_mount_does_not_raise(): ) module.run_mount( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, mount_arguments=mount_arguments, diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index db9c1247..7af7ea77 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -13,7 +13,7 @@ def test_run_prune_calls_hooks_for_configured_repository(): module.run_prune( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, @@ -38,7 +38,7 @@ def test_run_prune_runs_with_selected_repository(): module.run_prune( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, diff --git a/tests/unit/actions/test_rcreate.py b/tests/unit/actions/test_rcreate.py index 78d4af1f..b77fa757 100644 --- a/tests/unit/actions/test_rcreate.py +++ b/tests/unit/actions/test_rcreate.py @@ -18,7 +18,7 @@ def test_run_rcreate_does_not_raise(): ) module.run_rcreate( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rcreate_arguments=arguments, @@ -45,7 +45,7 @@ def test_run_rcreate_bails_if_repository_does_not_match(): ) module.run_rcreate( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rcreate_arguments=arguments, diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index eaad6bf4..8255c87d 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -67,7 +67,7 @@ def test_collect_archive_database_names_parses_archive_paths(): ) archive_database_names = module.collect_archive_database_names( - repository='repo', + repository={'path': 'repo'}, archive='archive', location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), @@ -92,7 +92,7 @@ def test_collect_archive_database_names_parses_directory_format_archive_paths(): ) archive_database_names = module.collect_archive_database_names( - repository='repo', + repository={'path': 'repo'}, archive='archive', location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), @@ -113,7 +113,7 @@ def test_collect_archive_database_names_skips_bad_archive_paths(): ) archive_database_names = module.collect_archive_database_names( - repository='repo', + repository={'path': 'repo'}, archive='archive', location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), @@ -251,7 +251,7 @@ def test_run_restore_restores_each_database(): flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), @@ -273,7 +273,7 @@ def test_run_restore_bails_for_non_matching_repository(): flexmock(module).should_receive('restore_single_database').never() module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), @@ -345,7 +345,7 @@ def test_run_restore_restores_database_configured_with_all_name(): flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), @@ -417,7 +417,7 @@ def test_run_restore_skips_missing_database(): flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), @@ -483,7 +483,7 @@ def test_run_restore_restores_databases_from_different_hooks(): flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), diff --git a/tests/unit/actions/test_rinfo.py b/tests/unit/actions/test_rinfo.py index d789ef11..133e61ac 100644 --- a/tests/unit/actions/test_rinfo.py +++ b/tests/unit/actions/test_rinfo.py @@ -11,7 +11,7 @@ def test_run_rinfo_does_not_raise(): list( module.run_rinfo( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rinfo_arguments=rinfo_arguments, diff --git a/tests/unit/actions/test_rlist.py b/tests/unit/actions/test_rlist.py index 3da0f905..7f8b58aa 100644 --- a/tests/unit/actions/test_rlist.py +++ b/tests/unit/actions/test_rlist.py @@ -11,7 +11,7 @@ def test_run_rlist_does_not_raise(): list( module.run_rlist( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rlist_arguments=rlist_arguments, diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 19ac00de..14e7443e 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -409,7 +409,7 @@ def test_run_actions_runs_rcreate(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -431,7 +431,7 @@ def test_run_actions_runs_transfer(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -454,7 +454,7 @@ def test_run_actions_runs_create(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -477,7 +477,7 @@ def test_run_actions_runs_prune(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -499,7 +499,7 @@ def test_run_actions_runs_compact(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -522,7 +522,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -545,7 +545,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -567,7 +567,7 @@ def test_run_actions_runs_extract(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -589,7 +589,7 @@ def test_run_actions_runs_export_tar(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -611,7 +611,7 @@ def test_run_actions_runs_mount(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -633,7 +633,7 @@ def test_run_actions_runs_restore(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -656,7 +656,7 @@ def test_run_actions_runs_rlist(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -680,7 +680,7 @@ def test_run_actions_runs_list(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -704,7 +704,7 @@ def test_run_actions_runs_rinfo(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -728,7 +728,7 @@ def test_run_actions_runs_info(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -751,7 +751,7 @@ def test_run_actions_runs_break_lock(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -773,7 +773,7 @@ def test_run_actions_runs_borg(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -800,7 +800,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 821c3202..ff67aa12 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -69,27 +69,27 @@ from borgmatic.config import normalize as module ), ( {'location': {'repositories': ['foo@bar:/repo']}}, - {'location': {'repositories': ['ssh://foo@bar/repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': ''}]}}, True, ), ( {'location': {'repositories': ['foo@bar:repo']}}, - {'location': {'repositories': ['ssh://foo@bar/./repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/./repo', 'label': ''}]}}, True, ), ( {'location': {'repositories': ['foo@bar:~/repo']}}, - {'location': {'repositories': ['ssh://foo@bar/~/repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo', 'label': ''}]}}, True, ), ( {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, - {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo', 'label': ''}]}}, False, ), ( {'location': {'repositories': ['file:///repo']}}, - {'location': {'repositories': ['/repo']}}, + {'location': {'repositories': [{'path': '/repo', 'label': ''}]}}, False, ), ), From aeaf69f49e0081a2b3c835d86d5206a550357155 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 01:34:03 +0530 Subject: [PATCH 14/94] pass all tests --- borgmatic/config/schema.yaml | 2 +- tests/end-to-end/test_borgmatic.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 14d028b7..e24b0eca 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -59,7 +59,7 @@ properties: used as the label. example: backupserver example: - - path: ssh://user@backupserver/./{fqdn} + - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver - path: /mnt/backup label: local diff --git a/tests/end-to-end/test_borgmatic.py b/tests/end-to-end/test_borgmatic.py index c2d10291..86cd83c2 100644 --- a/tests/end-to-end/test_borgmatic.py +++ b/tests/end-to-end/test_borgmatic.py @@ -19,9 +19,8 @@ def generate_configuration(config_path, repository_path): open(config_path) .read() .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) - .replace('- ssh://user@backupserver/./{fqdn}', '') - .replace('- /var/local/backups/local.borg', '') - .replace('- /home/user/path with spaces', '') + .replace('- path: /mnt/backup', '') + .replace('label: local', '') .replace('- /home', '- {}'.format(config_path)) .replace('- /etc', '') .replace('- /var/log/syslog*', '') From d6dfb8753a0a09cc2998fabd93f8fad78efd4850 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 01:50:47 +0530 Subject: [PATCH 15/94] reformat --- borgmatic/actions/compact.py | 8 +------- borgmatic/actions/export_tar.py | 4 +++- borgmatic/actions/restore.py | 4 +++- borgmatic/commands/borgmatic.py | 2 +- tests/end-to-end/test_borgmatic.py | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index e3018af5..95334c52 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -52,13 +52,7 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover -<<<<<<< HEAD - logger.info( - '{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository['path']) - ) -======= - logger.info(f'{repository}: Skipping compact (only available/needed in Borg 1.2+)') ->>>>>>> f42890430c59a40a17d9a68a193d6a09674770cb + logger.info(f'{repository["path"]}: Skipping compact (only available/needed in Borg 1.2+)') borgmatic.hooks.command.execute_hook( hooks.get('after_compact'), hooks.get('umask'), diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index 37a870a8..ff9f31ba 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -22,7 +22,9 @@ def run_export_tar( if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, export_tar_arguments.repository ): - logger.info(f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file') + logger.info( + f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file' + ) borgmatic.borg.export_tar.export_tar_archive( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index e718e74e..bbbb4c74 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -255,7 +255,9 @@ def run_restore( ): return - logger.info(f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}') + logger.info( + f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}' + ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 86f047ec..7612970c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -169,7 +169,7 @@ def run_configuration(config_filename, config, arguments): return encountered_error = error - yield from log_error_records(f'{repository_path}: Error pinging monitor', error) + yield from log_error_records(f'{repository["path"]}: Error pinging monitor', error) if not encountered_error: try: diff --git a/tests/end-to-end/test_borgmatic.py b/tests/end-to-end/test_borgmatic.py index c1653d2f..5e915f00 100644 --- a/tests/end-to-end/test_borgmatic.py +++ b/tests/end-to-end/test_borgmatic.py @@ -19,7 +19,7 @@ def generate_configuration(config_path, repository_path): .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) .replace('- path: /mnt/backup', '') .replace('label: local', '') - .replace('- /home', '- {}'.format(config_path)) + .replace('- /home', f'- {config_path}') .replace('- /etc', '') .replace('- /var/log/syslog*', '') + 'storage:\n encryption_passphrase: "test"' From af95134cd2ec7a779600247be818bf180aa8de7b Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 02:03:36 +0530 Subject: [PATCH 16/94] add test for complex constant --- borgmatic/config/load.py | 4 +++- tests/integration/config/test_load.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index e379537a..32181126 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -1,4 +1,5 @@ import functools +import json import logging import os @@ -104,7 +105,8 @@ def load_configuration(filename): config = yaml.load(file_contents) if config and 'constants' in config: for key, value in config['constants'].items(): - file_contents = file_contents.replace(f'{{{key}}}', str(value)) + value = json.dumps(value) + file_contents = file_contents.replace(f'{{{key}}}', value) config = yaml.load(file_contents) del config['constants'] return config diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index 5b8ee9b0..3382d734 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -30,6 +30,21 @@ def test_load_configuration_replaces_constants(): assert module.load_configuration('config.yaml') == {'key': 'value'} +def test_load_configuration_replaces_complex_constants(): + builtins = flexmock(sys.modules['builtins']) + config_file = io.StringIO( + ''' + constants: + key: + subkey: value + key: {key} + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + assert module.load_configuration('config.yaml') == {'key': {'subkey': 'value'}} + + def test_load_configuration_inlines_include_relative_to_current_directory(): builtins = flexmock(sys.modules['builtins']) flexmock(module.os).should_receive('getcwd').and_return('/tmp') From 78e8bb6c8c0d47d491f7a9a9cb1a4d892ef53963 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 02:08:52 +0530 Subject: [PATCH 17/94] reformat --- borgmatic/config/load.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 32181126..01ceab2b 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -100,8 +100,8 @@ def load_configuration(filename): yaml = ruamel.yaml.YAML(typ='safe') yaml.Constructor = Include_constructor_with_include_directory - with open(filename) as f: - file_contents = f.read() + with open(filename) as file: + file_contents = file.read() config = yaml.load(file_contents) if config and 'constants' in config: for key, value in config['constants'].items(): From 61ce6f04731a2f146acd375e8494d77d2cbd8874 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 02:44:56 +0530 Subject: [PATCH 18/94] fix: docs cli reference create spelling --- borgmatic/commands/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index d5dc6af4..bb89d82d 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -225,7 +225,7 @@ def make_parsers(): subparsers = top_level_parser.add_subparsers( title='actions', metavar='', - help='Specify zero or more actions. Defaults to creat, prune, compact, and check. Use --help with action for details:', + help='Specify zero or more actions. Defaults to create, prune, compact, and check. Use --help with action for details:', ) rcreate_parser = subparsers.add_parser( 'rcreate', From 9851abc2e105fd3c42ab447fcfbcf034cd208c91 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 24 Mar 2023 15:18:49 -0700 Subject: [PATCH 19/94] Add documentation on backing up a database running in a container (#649). --- NEWS | 2 ++ docs/how-to/backup-your-databases.md | 48 +++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index bc5f431e..2bbd7541 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,8 @@ * #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in borgmatic's storage configuration. * #623: Fix confusing message when an error occurs running actions for a configuration file. + * #649: Add documentation on backing up a database running in a container: + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers * #655: Fix error when databases are configured and a source directory doesn't exist. * Add code style plugins to enforce use of Python f-strings and prevent single-letter variables. To join in the pedantry, refresh your test environment with "tox --recreate". diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index bc21b659..3f819170 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -136,6 +136,53 @@ hooks: format: sql ``` +### Containers + +If your database is running within a Docker container and borgmatic is too, no +problem—simply configure borgmatic to connect to the container's name on its +exposed port. For instance: + +```yaml +hooks: + postgresql_databases: + - name: users + hostname: your-database-container-name + port: 5433 + username: postgres + password: trustsome1 +``` + +But what if borgmatic is running on the host? You can still connect to a +database container if its ports are properly exposed to the host. For +instance, when running the database container with Docker, you can specify +`--publish 127.0.0.1:5433:5432` so that it exposes the container's port 5432 +to port 5433 on the host (only reachable on localhost, in this case). Or the +same thing with Docker Compose: + +```yaml +services: + your-database-container-name: + image: postgres + ports: + - 127.0.0.1:5433:5432 +``` + +And then you can connect to the database from borgmatic running on the host: + +```yaml +hooks: + postgresql_databases: + - name: users + hostname: 127.0.0.1 + port: 5433 + username: postgres + password: trustsome1 +``` + +Of course, alter the ports in these examples to suit your particular database +system. + + ### No source directories New in version 1.7.1 If you @@ -154,7 +201,6 @@ hooks: ``` - ### External passwords If you don't want to keep your database passwords in your borgmatic From 739a58fe4773ddf6696f29659123613061bce7a3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 24 Mar 2023 16:24:00 -0700 Subject: [PATCH 20/94] Rename scripts/run-full-dev-tests to scripts/run-end-to-end-dev-tests and make it run end-to-end tests only. --- NEWS | 2 ++ docs/how-to/develop-on-borgmatic.md | 2 +- .../{run-full-dev-tests => run-end-to-end-dev-tests} | 0 scripts/run-full-tests | 10 +++++++--- tests/end-to-end/docker-compose.yaml | 4 ++-- 5 files changed, 12 insertions(+), 6 deletions(-) rename scripts/{run-full-dev-tests => run-end-to-end-dev-tests} (100%) diff --git a/NEWS b/NEWS index 2bbd7541..9badeacc 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,8 @@ * #655: Fix error when databases are configured and a source directory doesn't exist. * Add code style plugins to enforce use of Python f-strings and prevent single-letter variables. To join in the pedantry, refresh your test environment with "tox --recreate". + * Rename scripts/run-full-dev-tests to scripts/run-end-to-end-dev-tests and make it run end-to-end + tests only. Continue using tox to run unit and integration tests. 1.7.9 * #295: Add a SQLite database dump/restore hook. diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index fbc1d244..d4315457 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -87,7 +87,7 @@ If you would like to run the full test suite, first install Docker and [Docker Compose](https://docs.docker.com/compose/install/). Then run: ```bash -scripts/run-full-dev-tests +scripts/run-end-to-end-dev-tests ``` Note that this scripts assumes you have permission to run Docker. If you diff --git a/scripts/run-full-dev-tests b/scripts/run-end-to-end-dev-tests similarity index 100% rename from scripts/run-full-dev-tests rename to scripts/run-end-to-end-dev-tests diff --git a/scripts/run-full-tests b/scripts/run-full-tests index 58e5ac65..cbd824ce 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -3,12 +3,12 @@ # This script installs test dependencies and runs all tests, including end-to-end tests. It # is designed to run inside a test container, and presumes that other test infrastructure like # databases are already running. Therefore, on a developer machine, you should not run this script -# directly. Instead, run scripts/run-full-dev-tests +# directly. Instead, run scripts/run-end-to-end-dev-tests # # For more information, see: # https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/ -set -e +set -ex apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \ py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite @@ -17,5 +17,9 @@ apk add --no-cache py3-typed-ast py3-regex || true python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 pip3 install --ignore-installed tox==3.25.1 export COVERAGE_FILE=/tmp/.coverage -tox --workdir /tmp/.tox --sitepackages + +if [ "$1" != "--end-to-end-only" ] ; then + tox --workdir /tmp/.tox --sitepackages +fi + tox --workdir /tmp/.tox --sitepackages -e end-to-end diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 094ac8d3..090cf12c 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -23,8 +23,8 @@ services: - "/app/borgmatic.egg-info" tty: true working_dir: /app - command: - - /app/scripts/run-full-tests + entrypoint: /app/scripts/run-full-tests + command: --end-to-end-only depends_on: - postgresql - mysql From d14a8df71aae0a282708127dd1be1279154b5dac Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 24 Mar 2023 22:43:10 -0700 Subject: [PATCH 21/94] Hide obnoxious ruamel.yaml warnings during test runs. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d8d28f89..67b989dc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ description_file=README.md testpaths = tests addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end filterwarnings = - ignore:Coverage disabled.*:pytest.PytestWarning + ignore:Deprecated call to `pkg_resources.declare_namespace\('ruamel'\)`.*:DeprecationWarning [flake8] ignore = E501,W503 From a082cb87cb22c3b75ddfb9df4547d5e1a636bb40 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 12:12:56 +0530 Subject: [PATCH 22/94] fix: replace primitive values in config without quotes --- borgmatic/config/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 01ceab2b..9cc665ae 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -106,7 +106,7 @@ def load_configuration(filename): if config and 'constants' in config: for key, value in config['constants'].items(): value = json.dumps(value) - file_contents = file_contents.replace(f'{{{key}}}', value) + file_contents = file_contents.replace(f'{{{key}}}', value.strip('"')) config = yaml.load(file_contents) del config['constants'] return config From 19e95628c3ed57451bfc2c9a1421688a641f5211 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 24 Mar 2023 23:47:05 -0700 Subject: [PATCH 23/94] Add documentation and NEWS for custom constants feature (#612). --- NEWS | 3 ++ borgmatic/commands/borgmatic.py | 2 +- borgmatic/config/load.py | 3 ++ docs/how-to/make-per-application-backups.md | 57 +++++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 9badeacc..500a2cb5 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,9 @@ * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" option in borgmatic's location configuration. * #576: Add support for "file://" paths within "repositories" option. + * #612: Define and use custom constants in borgmatic configuration files. See the documentation for + more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#constants * #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in borgmatic's storage configuration. * #623: Fix confusing message when an error occurs running actions for a configuration file. diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 73bde94b..61b83951 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -581,7 +581,7 @@ def collect_configuration_run_summary_logs(configs, arguments): if not configs: yield from log_error_records( - r"{' '.join(arguments['global'].config_paths)}: No valid configuration files found", + f"{' '.join(arguments['global'].config_paths)}: No valid configuration files found", ) return diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 9cc665ae..f3c45d52 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -103,12 +103,15 @@ def load_configuration(filename): with open(filename) as file: file_contents = file.read() config = yaml.load(file_contents) + if config and 'constants' in config: for key, value in config['constants'].items(): value = json.dumps(value) file_contents = file_contents.replace(f'{{{key}}}', value.strip('"')) + config = yaml.load(file_contents) del config['constants'] + return config diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 2bf099e6..5641213e 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -255,3 +255,60 @@ Be sure to quote your overrides if they contain spaces or other characters that your shell may interpret. An alternate to command-line overrides is passing in your values via [environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). + + +## Constants + +New in version 1.7.10 Another +tool is borgmatic's support for defining custom constants. This is similar to +the [variable interpolation +feature](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation) +for command hooks, but the constants feature lets you substitute your own +custom values into anywhere in the entire configuration file. (Constants don't +work across includes or separate configuration files though.) + +Here's an example usage: + +```yaml +constants: + user: foo + my_prefix: bar- + +location: + source_directories: + - /home/{user}/.config + - /home/{user}/.ssh + ... + +storage: + archive_name_format: '{my_prefix}{now}' + +retention: + prefix: {my_prefix} + +consistency: + prefix: {my_prefix} +``` + +In this example, when borgmatic runs, all instances of `{user}` get replaced +with `foo` and all instances of `{my_prefix}` get replaced with `bar-`. (And +in this particular example, `{now}` doesn't get replaced with anything, but +gets passed directly to Borg.) After substitution, the logical result looks +something like this: + +```yaml +location: + source_directories: + - /home/foo/.config + - /home/foo/.ssh + ... + +storage: + archive_name_format: 'bar-{now}' + +retention: + prefix: bar- + +consistency: + prefix: bar- +``` From 8c0eea72296c6c5c98cd392107381d47d0c878e1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 25 Mar 2023 08:56:25 -0700 Subject: [PATCH 24/94] Add additional documentation link to environment variable feature. Rename constants section. --- NEWS | 2 +- docs/how-to/make-per-application-backups.md | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 500a2cb5..7d02c0e2 100644 --- a/NEWS +++ b/NEWS @@ -4,7 +4,7 @@ * #576: Add support for "file://" paths within "repositories" option. * #612: Define and use custom constants in borgmatic configuration files. See the documentation for more information: - https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#constants + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#constant-interpolation * #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in borgmatic's storage configuration. * #623: Fix confusing message when an error occurs running actions for a configuration file. diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 5641213e..e5ba037f 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -257,7 +257,7 @@ that your shell may interpret. An alternate to command-line overrides is passing in your values via [environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). -## Constants +## Constant interpolation New in version 1.7.10 Another tool is borgmatic's support for defining custom constants. This is similar to @@ -312,3 +312,6 @@ retention: consistency: prefix: bar- ``` + +An alternate to constants is passing in your values via [environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). From f56fdab7a9b10309b464eb85fe94d4a75b1a4e93 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 25 Mar 2023 17:08:17 -0700 Subject: [PATCH 25/94] Add troubleshooting documentation on PostgreSQL/MySQL authentication errors. --- docs/how-to/backup-your-databases.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 3f819170..af60253c 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -380,6 +380,23 @@ dumps with any database system. ## Troubleshooting +### PostgreSQL/MySQL authentication errors + +With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors +when borgmatic tries to connect to your database, a natural reaction is to +increase your borgmatic verbosity with `--verbosity 2` and go looking in the +logs. You'll notice however that your database password does not show up in +the logs. This is likely not the cause of the authentication problem unless +you mistyped your password, however; borgmatic passes your password to the +database via an environment variable that does not appear in the logs. + +The cause of an authentication error is often on the database side—in the +configuration of which users are allowed to connect and how they are +authenticated. For instance, with PostgreSQL, check your +[pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) +file for that configuration. + + ### MySQL table lock errors If you encounter table lock errors during a database dump with MySQL/MariaDB, From b511e679ae7792257afa7201dfc55314df72bc2e Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sun, 26 Mar 2023 16:59:29 +0530 Subject: [PATCH 26/94] remove optional label for repos from tests --- borgmatic/commands/borgmatic.py | 5 +- borgmatic/config/normalize.py | 19 +--- borgmatic/config/schema.yaml | 2 +- borgmatic/config/validate.py | 2 +- tests/integration/config/test_validate.py | 18 +-- tests/unit/commands/test_borgmatic.py | 130 +++++++++++----------- tests/unit/config/test_normalize.py | 10 +- 7 files changed, 86 insertions(+), 100 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 7612970c..0954ba63 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -107,8 +107,7 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository, retry_num = repo_queue.get() - if isinstance(repository, str): - repository = {'path': repository} + logger.debug(f'{repository["path"]}: Running actions for repository') timeout = retry_num * retry_wait if timeout: logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') @@ -261,8 +260,6 @@ def run_actions( invalid. ''' add_custom_log_levels() - if isinstance(repository, str): - repository = {'path': repository} repository_path = os.path.expanduser(repository['path']) global_arguments = arguments['global'] dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index eb2ed013..bcf088ac 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -80,17 +80,11 @@ def normalize(config_filename, config): updated_repository_path = os.path.abspath( repository_path.partition('file://')[-1] ) - config['location']['repositories'].append( - { - 'path': updated_repository_path, - 'label': repository_dict.get('label', ''), - } + dict(repository_dict, path=updated_repository_path,) ) elif repository_path.startswith('ssh://'): - config['location']['repositories'].append( - {'path': repository_path, 'label': repository_dict.get('label', '')} - ) + config['location']['repositories'].append(repository_dict) else: rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( @@ -103,14 +97,9 @@ def normalize(config_filename, config): ) ) config['location']['repositories'].append( - { - 'path': rewritten_repository_path, - 'label': repository_dict.get('label', ''), - } + dict(repository_dict, path=rewritten_repository_path,) ) else: - config['location']['repositories'].append( - {'path': repository_path, 'label': repository_dict.get('label', '')} - ) + config['location']['repositories'].append(repository_dict) return logs diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index e24b0eca..4a494199 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -53,7 +53,7 @@ properties: type: string description: | Optional label for the repository. This - can be used with the --repository option + can be used with the --repository flag to select a repository to backup to. If not specified, the repository path is used as the label. diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index e40603fe..abcfe3d2 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -138,7 +138,7 @@ def normalize_repository_path(repository): def repositories_match(first, second): ''' Given two repository dicts with keys 'path' (relative and/or absolute), - and 'label', return whether they match. + and 'label', or two repository paths, return whether they match. ''' if isinstance(first, str): first = {'path': first, 'label': first} diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 31a85cd1..67e510ab 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -65,7 +65,7 @@ def test_parse_configuration_transforms_file_into_mapping(): assert config == { 'location': { 'source_directories': ['/home', '/etc'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], + 'repositories': [{'path': 'hostname.borg'}], }, 'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60}, 'consistency': {'checks': [{'name': 'repository'}, {'name': 'archives'}]}, @@ -92,7 +92,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): assert config == { 'location': { 'source_directories': [f'/home/{string.punctuation}'], - 'repositories': [{'path': 'test.borg', 'label': ''}], + 'repositories': [{'path': 'test.borg'}], } } assert logs == [] @@ -154,10 +154,7 @@ def test_parse_configuration_inlines_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': { - 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], - }, + 'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]}, 'retention': {'keep_daily': 7, 'keep_hourly': 24}, } assert logs == [] @@ -191,10 +188,7 @@ def test_parse_configuration_merges_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': { - 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], - }, + 'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]}, 'retention': {'keep_daily': 1, 'keep_hourly': 24}, } assert logs == [] @@ -256,7 +250,7 @@ def test_parse_configuration_applies_overrides(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], + 'repositories': [{'path': 'hostname.borg'}], 'local_path': 'borg2', } } @@ -282,7 +276,7 @@ def test_parse_configuration_applies_normalization(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], + 'repositories': [{'path': 'hostname.borg'}], 'exclude_if_present': ['.nobackup'], } } diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 14e7443e..b9a16f35 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -15,7 +15,7 @@ def test_run_configuration_runs_actions_for_each_repository(): flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return( expected_results[1:] ) - config = {'location': {'repositories': ['foo', 'bar']}} + config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}} arguments = {'global': flexmock(monitoring_verbosity=1)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -75,7 +75,7 @@ def test_run_configuration_logs_actions_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -91,7 +91,7 @@ def test_run_configuration_bails_for_actions_soft_failure(): flexmock(module).should_receive('run_actions').and_raise(error) flexmock(module).should_receive('log_error_records').never() flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -108,7 +108,7 @@ def test_run_configuration_logs_monitor_log_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -126,7 +126,7 @@ def test_run_configuration_bails_for_monitor_log_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -143,7 +143,7 @@ def test_run_configuration_logs_monitor_finish_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -161,7 +161,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -178,7 +178,7 @@ def test_run_configuration_logs_on_error_hook_error(): expected_results[:1] ).and_return(expected_results[1:]) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -194,7 +194,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -209,7 +209,7 @@ def test_run_configuration_retries_soft_error(): flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} + config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == [] @@ -231,7 +231,7 @@ def test_run_configuration_retries_hard_error(): flexmock(module).should_receive('log_error_records').with_args( 'foo: Error running actions for repository', OSError, ).and_return(error_logs) - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} + config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -249,41 +249,41 @@ def test_run_configuration_repos_ordered(): flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError ).and_return(expected_results[1:]).ordered() - config = {'location': {'repositories': ['foo', 'bar']}} + config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results -def test_run_configuration_retries_round_robbin(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') - flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) - flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', - OSError, - levelno=logging.WARNING, - log_command_error_output=True, - ).and_return([flexmock()]).ordered() - flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', - OSError, - levelno=logging.WARNING, - log_command_error_output=True, - ).and_return([flexmock()]).ordered() - foo_error_logs = [flexmock()] - flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', OSError - ).and_return(foo_error_logs).ordered() - bar_error_logs = [flexmock()] - flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', OSError - ).and_return(bar_error_logs).ordered() - config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) - assert results == foo_error_logs + bar_error_logs +# def test_run_configuration_retries_round_robin(): +# flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) +# flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) +# flexmock(module.command).should_receive('execute_hook') +# flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) +# flexmock(module).should_receive('log_error_records').with_args( +# 'foo: Error running actions for repository', +# OSError, +# levelno=logging.WARNING, +# log_command_error_output=True, +# ).and_return([flexmock()]).ordered() +# flexmock(module).should_receive('log_error_records').with_args( +# 'bar: Error running actions for repository', +# OSError, +# levelno=logging.WARNING, +# log_command_error_output=True, +# ).and_return([flexmock()]).ordered() +# foo_error_logs = [flexmock()] +# flexmock(module).should_receive('log_error_records').with_args( +# 'foo: Error running actions for repository', OSError +# ).and_return(foo_error_logs).ordered() +# bar_error_logs = [flexmock()] +# flexmock(module).should_receive('log_error_records').with_args( +# 'bar: Error running actions for repository', OSError +# ).and_return(bar_error_logs).ordered() +# config = {'location': {'repositories': [{'path':'foo','path':'bar'}]}, 'storage': {'retries': 1}} +# arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} +# results = list(module.run_configuration('test.yaml', config, arguments)) +# assert results == foo_error_logs + bar_error_logs def test_run_configuration_retries_one_passes(): @@ -309,7 +309,10 @@ def test_run_configuration_retries_one_passes(): flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError ).and_return(error_logs).ordered() - config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} + config = { + 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, + 'storage': {'retries': 1}, + } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -348,7 +351,10 @@ def test_run_configuration_retry_wait(): flexmock(module).should_receive('log_error_records').with_args( 'foo: Error running actions for repository', OSError ).and_return(error_logs).ordered() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}} + config = { + 'location': {'repositories': [{'path': 'foo'}]}, + 'storage': {'retries': 3, 'retry_wait': 10}, + } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -384,7 +390,7 @@ def test_run_configuration_retries_timeout_multiple_repos(): 'bar: Error running actions for repository', OSError ).and_return(error_logs).ordered() config = { - 'location': {'repositories': ['foo', 'bar']}, + 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, 'storage': {'retries': 1, 'retry_wait': 10}, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -409,7 +415,7 @@ def test_run_actions_runs_rcreate(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -431,7 +437,7 @@ def test_run_actions_runs_transfer(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -454,7 +460,7 @@ def test_run_actions_runs_create(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -477,7 +483,7 @@ def test_run_actions_runs_prune(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -499,7 +505,7 @@ def test_run_actions_runs_compact(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -522,7 +528,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -545,7 +551,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -567,7 +573,7 @@ def test_run_actions_runs_extract(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -589,7 +595,7 @@ def test_run_actions_runs_export_tar(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -611,7 +617,7 @@ def test_run_actions_runs_mount(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -633,7 +639,7 @@ def test_run_actions_runs_restore(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -656,7 +662,7 @@ def test_run_actions_runs_rlist(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -680,7 +686,7 @@ def test_run_actions_runs_list(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -704,7 +710,7 @@ def test_run_actions_runs_rinfo(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -728,7 +734,7 @@ def test_run_actions_runs_info(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -751,7 +757,7 @@ def test_run_actions_runs_break_lock(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -773,7 +779,7 @@ def test_run_actions_runs_borg(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -800,7 +806,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index ff67aa12..fe9e7ac1 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -69,27 +69,27 @@ from borgmatic.config import normalize as module ), ( {'location': {'repositories': ['foo@bar:/repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/repo'}]}}, True, ), ( {'location': {'repositories': ['foo@bar:repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/./repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/./repo'}]}}, True, ), ( {'location': {'repositories': ['foo@bar:~/repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo'}]}}, True, ), ( {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo'}]}}, False, ), ( {'location': {'repositories': ['file:///repo']}}, - {'location': {'repositories': [{'path': '/repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': '/repo'}]}}, False, ), ), From a136fda92d599f0dd6ecbf48e82f6bc0771ef956 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sun, 26 Mar 2023 23:35:47 +0530 Subject: [PATCH 27/94] check all tests --- tests/unit/commands/test_borgmatic.py | 61 ++++++++++++++------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index b9a16f35..11e879df 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -255,35 +255,38 @@ def test_run_configuration_repos_ordered(): assert results == expected_results -# def test_run_configuration_retries_round_robin(): -# flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) -# flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) -# flexmock(module.command).should_receive('execute_hook') -# flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) -# flexmock(module).should_receive('log_error_records').with_args( -# 'foo: Error running actions for repository', -# OSError, -# levelno=logging.WARNING, -# log_command_error_output=True, -# ).and_return([flexmock()]).ordered() -# flexmock(module).should_receive('log_error_records').with_args( -# 'bar: Error running actions for repository', -# OSError, -# levelno=logging.WARNING, -# log_command_error_output=True, -# ).and_return([flexmock()]).ordered() -# foo_error_logs = [flexmock()] -# flexmock(module).should_receive('log_error_records').with_args( -# 'foo: Error running actions for repository', OSError -# ).and_return(foo_error_logs).ordered() -# bar_error_logs = [flexmock()] -# flexmock(module).should_receive('log_error_records').with_args( -# 'bar: Error running actions for repository', OSError -# ).and_return(bar_error_logs).ordered() -# config = {'location': {'repositories': [{'path':'foo','path':'bar'}]}, 'storage': {'retries': 1}} -# arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} -# results = list(module.run_configuration('test.yaml', config, arguments)) -# assert results == foo_error_logs + bar_error_logs +def test_run_configuration_retries_round_robin(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) + flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) + flexmock(module).should_receive('log_error_records').with_args( + 'foo: Error running actions for repository', + OSError, + levelno=logging.WARNING, + log_command_error_output=True, + ).and_return([flexmock()]).ordered() + flexmock(module).should_receive('log_error_records').with_args( + 'bar: Error running actions for repository', + OSError, + levelno=logging.WARNING, + log_command_error_output=True, + ).and_return([flexmock()]).ordered() + foo_error_logs = [flexmock()] + flexmock(module).should_receive('log_error_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(foo_error_logs).ordered() + bar_error_logs = [flexmock()] + flexmock(module).should_receive('log_error_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(bar_error_logs).ordered() + config = { + 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, + 'storage': {'retries': 1}, + } + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == foo_error_logs + bar_error_logs def test_run_configuration_retries_one_passes(): From ec9def4e71a98d876fdfdbbbe718b8965452491e Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sun, 26 Mar 2023 23:52:25 +0530 Subject: [PATCH 28/94] rename repository arg to repository_path in all borg actions --- borgmatic/borg/borg.py | 6 +- borgmatic/borg/break_lock.py | 4 +- borgmatic/borg/check.py | 10 ++-- borgmatic/borg/compact.py | 6 +- borgmatic/borg/create.py | 12 ++-- borgmatic/borg/export_tar.py | 6 +- borgmatic/borg/extract.py | 8 ++- borgmatic/borg/flags.py | 10 ++-- borgmatic/borg/info.py | 4 +- borgmatic/borg/list.py | 31 +++++----- borgmatic/borg/mount.py | 8 +-- borgmatic/borg/prune.py | 4 +- borgmatic/borg/rcreate.py | 10 ++-- borgmatic/borg/rinfo.py | 4 +- borgmatic/borg/rlist.py | 24 +++++--- borgmatic/borg/transfer.py | 4 +- tests/unit/borg/test_borg.py | 43 +++++++++----- tests/unit/borg/test_break_lock.py | 12 ++-- tests/unit/borg/test_check.py | 28 ++++----- tests/unit/borg/test_compact.py | 30 ++++++---- tests/unit/borg/test_create.py | 92 +++++++++++++++--------------- tests/unit/borg/test_export_tar.py | 26 ++++----- tests/unit/borg/test_extract.py | 14 ++--- tests/unit/borg/test_flags.py | 10 ++-- tests/unit/borg/test_info.py | 24 ++++---- tests/unit/borg/test_list.py | 58 +++++++++---------- tests/unit/borg/test_mount.py | 22 +++---- tests/unit/borg/test_prune.py | 22 +++---- tests/unit/borg/test_rcreate.py | 30 +++++----- tests/unit/borg/test_rinfo.py | 20 +++---- tests/unit/borg/test_rlist.py | 30 +++++----- tests/unit/borg/test_transfer.py | 24 ++++---- 32 files changed, 339 insertions(+), 297 deletions(-) diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index 460d9d68..f19d6555 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -13,7 +13,7 @@ BORG_SUBCOMMANDS_WITHOUT_REPOSITORY = (('debug', 'info'), ('debug', 'convert-pro def run_arbitrary_borg( - repository, + repository_path, storage_config, local_borg_version, options, @@ -44,10 +44,10 @@ def run_arbitrary_borg( repository_archive_flags = () elif archive: repository_archive_flags = flags.make_repository_archive_flags( - repository, archive, local_borg_version + repository_path, archive, local_borg_version ) else: - repository_archive_flags = flags.make_repository_flags(repository, local_borg_version) + repository_archive_flags = flags.make_repository_flags(repository_path, local_borg_version) full_command = ( (local_path,) diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 820b1c56..2dff31ec 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) def break_lock( - repository, storage_config, local_borg_version, local_path='borg', remote_path=None, + repository_path, storage_config, local_borg_version, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage configuration dict, the local Borg version, @@ -24,7 +24,7 @@ def break_lock( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) borg_environment = environment.make_environment(storage_config) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 2914d83b..993a3c0a 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -243,7 +243,7 @@ def read_check_time(path): def check_archives( - repository, + repository_path, location_config, storage_config, consistency_config, @@ -268,7 +268,7 @@ def check_archives( try: borg_repository_id = json.loads( rinfo.display_repository_info( - repository, + repository_path, storage_config, local_borg_version, argparse.Namespace(json=True), @@ -277,7 +277,7 @@ def check_archives( ) )['repository']['id'] except (json.JSONDecodeError, KeyError): - raise ValueError(f'Cannot determine Borg repository ID for {repository}') + raise ValueError(f'Cannot determine Borg repository ID for {repository_path}') checks = filter_checks_on_frequency( location_config, @@ -310,7 +310,7 @@ def check_archives( + verbosity_flags + (('--progress',) if progress else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) borg_environment = environment.make_environment(storage_config) @@ -329,6 +329,6 @@ def check_archives( if 'extract' in checks: extract.extract_last_archive_dry_run( - storage_config, local_borg_version, repository, lock_wait, local_path, remote_path + storage_config, local_borg_version, repository_path, lock_wait, local_path, remote_path ) write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract')) diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index 847ed26b..0e9d3e89 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def compact_segments( dry_run, - repository, + repository_path, storage_config, local_borg_version, local_path='borg', @@ -36,11 +36,11 @@ def compact_segments( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) if dry_run: - logging.info(f'{repository}: Skipping compact (dry run)') + logging.info(f'{repository_path}: Skipping compact (dry run)') return execute_command( diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index d557a6ab..58c3509b 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -322,7 +322,7 @@ def check_all_source_directories_exist(source_directories): def create_archive( dry_run, - repository, + repository_path, location_config, storage_config, local_borg_version, @@ -411,7 +411,7 @@ def create_archive( if stream_processes and location_config.get('read_special') is False: logger.warning( - f'{repository}: Ignoring configured "read_special" value of false, as true is needed for database hooks.' + f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.' ) create_command = ( @@ -446,7 +446,9 @@ def create_archive( ) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_archive_flags(repository, archive_name_format, local_borg_version) + + flags.make_repository_archive_flags( + repository_path, archive_name_format, local_borg_version + ) + (sources if not pattern_file else ()) ) @@ -466,7 +468,7 @@ def create_archive( # If database hooks are enabled (as indicated by streaming processes), exclude files that might # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True. if stream_processes and not location_config.get('read_special'): - logger.debug(f'{repository}: Collecting special file paths') + logger.debug(f'{repository_path}: Collecting special file paths') special_file_paths = collect_special_file_paths( create_command, local_path, @@ -477,7 +479,7 @@ def create_archive( if special_file_paths: logger.warning( - f'{repository}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}' + f'{repository_path}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}' ) exclude_file = write_pattern_file( expand_home_directories( diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 01d1b7ed..2b427f03 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def export_tar_archive( dry_run, - repository, + repository_path, archive, paths, destination_path, @@ -45,7 +45,7 @@ def export_tar_archive( + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + flags.make_repository_archive_flags(repository, archive, local_borg_version,) + + flags.make_repository_archive_flags(repository_path, archive, local_borg_version,) + (destination_path,) + (tuple(paths) if paths else ()) ) @@ -56,7 +56,7 @@ def export_tar_archive( output_log_level = logging.INFO if dry_run: - logging.info(f'{repository}: Skipping export to tar file (dry run)') + logging.info(f'{repository_path}: Skipping export to tar file (dry run)') return execute_command( diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 6c32f7f0..a95685cf 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) def extract_last_archive_dry_run( storage_config, local_borg_version, - repository, + repository_path, lock_wait=None, local_path='borg', remote_path=None, @@ -30,7 +30,7 @@ def extract_last_archive_dry_run( try: last_archive_name = rlist.resolve_archive_name( - repository, 'latest', storage_config, local_borg_version, local_path, remote_path + repository_path, 'latest', storage_config, local_borg_version, local_path, remote_path ) except ValueError: logger.warning('No archives found. Skipping extract consistency check.') @@ -44,7 +44,9 @@ def extract_last_archive_dry_run( + lock_wait_flags + verbosity_flags + list_flag - + flags.make_repository_archive_flags(repository, last_archive_name, local_borg_version) + + flags.make_repository_archive_flags( + repository_path, last_archive_name, local_borg_version + ) ) execute_command( diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 845e0ff3..5dcebf50 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -33,7 +33,7 @@ def make_flags_from_arguments(arguments, excludes=()): ) -def make_repository_flags(repository, local_borg_version): +def make_repository_flags(repository_path, local_borg_version): ''' Given the path of a Borg repository and the local Borg version, return Borg-version-appropriate command-line flags (as a tuple) for selecting that repository. @@ -42,17 +42,17 @@ def make_repository_flags(repository, local_borg_version): ('--repo',) if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) else () - ) + (repository,) + ) + (repository_path,) -def make_repository_archive_flags(repository, archive, local_borg_version): +def make_repository_archive_flags(repository_path, archive, local_borg_version): ''' Given the path of a Borg repository, an archive name or pattern, and the local Borg version, return Borg-version-appropriate command-line flags (as a tuple) for selecting that repository and archive. ''' return ( - ('--repo', repository, archive) + ('--repo', repository_path, archive) if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) - else (f'{repository}::{archive}',) + else (f'{repository_path}::{archive}',) ) diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index bcde24cd..6142104f 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def display_archives_info( - repository, + repository_path, storage_config, local_borg_version, info_arguments, @@ -49,7 +49,7 @@ def display_archives_info( + flags.make_flags_from_arguments( info_arguments, excludes=('repository', 'archive', 'prefix') ) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) + ( flags.make_flags('match-archives', info_arguments.archive) if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 916d17b0..908f8fef 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -21,7 +21,7 @@ MAKE_FLAGS_EXCLUDES = ( def make_list_command( - repository, + repository_path, storage_config, local_borg_version, list_arguments, @@ -52,10 +52,10 @@ def make_list_command( + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES) + ( flags.make_repository_archive_flags( - repository, list_arguments.archive, local_borg_version + repository_path, list_arguments.archive, local_borg_version ) if list_arguments.archive - else flags.make_repository_flags(repository, local_borg_version) + else flags.make_repository_flags(repository_path, local_borg_version) ) + (tuple(list_arguments.paths) if list_arguments.paths else ()) ) @@ -86,7 +86,7 @@ def make_find_paths(find_paths): def capture_archive_listing( - repository, + repository_path, archive, storage_config, local_borg_version, @@ -104,11 +104,11 @@ def capture_archive_listing( return tuple( execute_command_and_capture_output( make_list_command( - repository, + repository_path, storage_config, local_borg_version, argparse.Namespace( - repository=repository, + repository=repository_path, archive=archive, paths=[f'sh:{list_path}'], find_paths=None, @@ -126,7 +126,7 @@ def capture_archive_listing( def list_archive( - repository, + repository_path, storage_config, local_borg_version, list_arguments, @@ -149,7 +149,7 @@ def list_archive( ) rlist_arguments = argparse.Namespace( - repository=repository, + repository=repository_path, short=list_arguments.short, format=list_arguments.format, json=list_arguments.json, @@ -160,7 +160,12 @@ def list_archive( last=list_arguments.last, ) return rlist.list_repository( - repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path + repository_path, + storage_config, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ) if list_arguments.archive: @@ -181,7 +186,7 @@ def list_archive( # getting a list of archives to search. if list_arguments.find_paths and not list_arguments.archive: rlist_arguments = argparse.Namespace( - repository=repository, + repository=repository_path, short=True, format=None, json=None, @@ -196,7 +201,7 @@ def list_archive( archive_lines = tuple( execute_command_and_capture_output( rlist.make_rlist_command( - repository, + repository_path, storage_config, local_borg_version, rlist_arguments, @@ -213,7 +218,7 @@ def list_archive( # For each archive listed by Borg, run list on the contents of that archive. for archive in archive_lines: - logger.answer(f'{repository}: Listing archive {archive}') + logger.answer(f'{repository_path}: Listing archive {archive}') archive_arguments = copy.copy(list_arguments) archive_arguments.archive = archive @@ -224,7 +229,7 @@ def list_archive( setattr(archive_arguments, name, None) main_command = make_list_command( - repository, + repository_path, storage_config, local_borg_version, archive_arguments, diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 92d689b2..07a6c632 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) def mount_archive( - repository, + repository_path, archive, mount_point, paths, @@ -38,7 +38,7 @@ def mount_archive( + (('-o', options) if options else ()) + ( ( - flags.make_repository_flags(repository, local_borg_version) + flags.make_repository_flags(repository_path, local_borg_version) + ( ('--match-archives', archive) if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) @@ -47,9 +47,9 @@ def mount_archive( ) if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) else ( - flags.make_repository_archive_flags(repository, archive, local_borg_version) + flags.make_repository_archive_flags(repository_path, archive, local_borg_version) if archive - else flags.make_repository_flags(repository, local_borg_version) + else flags.make_repository_flags(repository_path, local_borg_version) ) ) + (mount_point,) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 5be85de2..d21ceee3 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -39,7 +39,7 @@ def make_prune_flags(retention_config, local_borg_version): def prune_archives( dry_run, - repository, + repository_path, storage_config, retention_config, local_borg_version, @@ -74,7 +74,7 @@ def prune_archives( + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) if stats or list_archives: diff --git a/borgmatic/borg/rcreate.py b/borgmatic/borg/rcreate.py index d3a8f7aa..7510529d 100644 --- a/borgmatic/borg/rcreate.py +++ b/borgmatic/borg/rcreate.py @@ -13,7 +13,7 @@ RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2 def create_repository( dry_run, - repository, + repository_path, storage_config, local_borg_version, encryption_mode, @@ -33,14 +33,14 @@ def create_repository( ''' try: rinfo.display_repository_info( - repository, + repository_path, storage_config, local_borg_version, argparse.Namespace(json=True), local_path, remote_path, ) - logger.info(f'{repository}: Repository already exists. Skipping creation.') + logger.info(f'{repository_path}: Repository already exists. Skipping creation.') return except subprocess.CalledProcessError as error: if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE: @@ -65,11 +65,11 @@ def create_repository( + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + (('--remote-path', remote_path) if remote_path else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) if dry_run: - logging.info(f'{repository}: Skipping repository creation (dry run)') + logging.info(f'{repository_path}: Skipping repository creation (dry run)') return # Do not capture output here, so as to support interactive prompts. diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index 7bc9a5e9..8a30b7f0 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def display_repository_info( - repository, + repository_path, storage_config, local_borg_version, rinfo_arguments, @@ -43,7 +43,7 @@ def display_repository_info( + flags.make_flags('remote-path', remote_path) + flags.make_flags('lock-wait', lock_wait) + (('--json',) if rinfo_arguments.json else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) extra_environment = environment.make_environment(storage_config) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 43bc28d6..8625363b 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def resolve_archive_name( - repository, archive, storage_config, local_borg_version, local_path='borg', remote_path=None + repository_path, + archive, + storage_config, + local_borg_version, + local_path='borg', + remote_path=None, ): ''' Given a local or remote repository path, an archive name, a storage config dict, a local Borg @@ -31,7 +36,7 @@ def resolve_archive_name( + flags.make_flags('lock-wait', lock_wait) + flags.make_flags('last', 1) + ('--short',) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) output = execute_command_and_capture_output( @@ -42,7 +47,7 @@ def resolve_archive_name( except IndexError: raise ValueError('No archives found in the repository') - logger.debug(f'{repository}: Latest archive is {latest_archive}') + logger.debug(f'{repository_path}: Latest archive is {latest_archive}') return latest_archive @@ -51,7 +56,7 @@ MAKE_FLAGS_EXCLUDES = ('repository', 'prefix') def make_rlist_command( - repository, + repository_path, storage_config, local_borg_version, rlist_arguments, @@ -92,12 +97,12 @@ def make_rlist_command( else () ) + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) def list_repository( - repository, + repository_path, storage_config, local_borg_version, rlist_arguments, @@ -113,7 +118,12 @@ def list_repository( borg_environment = environment.make_environment(storage_config) main_command = make_rlist_command( - repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path + repository_path, + storage_config, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ) if rlist_arguments.json: diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index bad02d06..29e205c7 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def transfer_archives( dry_run, - repository, + repository_path, storage_config, local_borg_version, transfer_arguments, @@ -38,7 +38,7 @@ def transfer_archives( transfer_arguments, excludes=('repository', 'source_repository', 'archive', 'match_archives'), ) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) + flags.make_flags('other-repo', transfer_arguments.source_repository) + flags.make_flags('dry-run', dry_run) ) diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 6eae5fb4..545da6c0 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -21,7 +21,10 @@ def test_run_arbitrary_borg_calls_borg_with_parameters(): ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['break-lock'], ) @@ -40,7 +43,10 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['break-lock'], ) @@ -59,7 +65,10 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['break-lock'], ) @@ -80,7 +89,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters( ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', options=['break-lock'], @@ -103,7 +112,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], @@ -125,7 +134,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], @@ -149,7 +158,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], @@ -171,7 +180,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['list', '--progress'], @@ -192,7 +201,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['--', 'break-lock'], @@ -213,7 +222,7 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=[], + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=[], ) @@ -231,7 +240,10 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository(): ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['key', 'export'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['key', 'export'], ) @@ -249,7 +261,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository() ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['debug', 'dump-manifest', 'path'], @@ -270,7 +282,10 @@ def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repositor ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['debug', 'info'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['debug', 'info'], ) @@ -288,7 +303,7 @@ def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_bor ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['debug', 'convert-profile', 'in', 'out'], diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 0663f93c..210aceeb 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -19,7 +19,7 @@ def test_break_lock_calls_borg_with_required_flags(): insert_execute_command_mock(('borg', 'break-lock', 'repo')) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', ) @@ -28,7 +28,7 @@ def test_break_lock_calls_borg_with_remote_path_flags(): insert_execute_command_mock(('borg', 'break-lock', '--remote-path', 'borg1', 'repo')) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', ) @@ -37,7 +37,7 @@ def test_break_lock_calls_borg_with_umask_flags(): insert_execute_command_mock(('borg', 'break-lock', '--umask', '0770', 'repo')) module.break_lock( - repository='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + repository_path='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', ) @@ -46,7 +46,7 @@ def test_break_lock_calls_borg_with_lock_wait_flags(): insert_execute_command_mock(('borg', 'break-lock', '--lock-wait', '5', 'repo')) module.break_lock( - repository='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + repository_path='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', ) @@ -56,7 +56,7 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', ) @@ -66,5 +66,5 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags(): insert_logging_mock(logging.DEBUG) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index ba82225c..75755565 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -370,7 +370,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -400,7 +400,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -435,7 +435,7 @@ def test_check_archives_calls_borg_with_parameters(checks): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -455,7 +455,7 @@ def test_check_archives_with_json_error_raises(): with pytest.raises(ValueError): module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -473,7 +473,7 @@ def test_check_archives_with_missing_json_keys_raises(): with pytest.raises(ValueError): module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -497,7 +497,7 @@ def test_check_archives_with_extract_check_calls_extract_only(): insert_execute_command_never() module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -521,7 +521,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -545,7 +545,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -563,7 +563,7 @@ def test_check_archives_without_any_checks_bails(): insert_execute_command_never() module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -589,7 +589,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -616,7 +616,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -643,7 +643,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={'lock_wait': 5}, consistency_config=consistency_config, @@ -670,7 +670,7 @@ def test_check_archives_with_retention_prefix(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -693,7 +693,7 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={'extra_borg_options': {'check': '--extra --options'}}, consistency_config=consistency_config, diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 36760f3c..60447db6 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -25,7 +25,7 @@ def test_compact_segments_calls_borg_with_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) module.compact_segments( - dry_run=False, repository='repo', storage_config={}, local_borg_version='1.2.3' + dry_run=False, repository_path='repo', storage_config={}, local_borg_version='1.2.3' ) @@ -35,7 +35,7 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.compact_segments( - repository='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False ) @@ -45,7 +45,7 @@ def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.compact_segments( - repository='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False ) @@ -53,7 +53,7 @@ def test_compact_segments_with_dry_run_skips_borg_call(): flexmock(module).should_receive('execute_command').never() module.compact_segments( - repository='repo', storage_config={}, local_borg_version='1.2.3', dry_run=True + repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=True ) @@ -63,7 +63,7 @@ def test_compact_segments_with_local_path_calls_borg_via_local_path(): module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', local_path='borg1', @@ -76,7 +76,7 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', @@ -89,7 +89,7 @@ def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', progress=True, @@ -102,7 +102,7 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', cleanup_commits=True, @@ -115,7 +115,7 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', threshold=20, @@ -128,7 +128,10 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.INFO) module.compact_segments( - dry_run=False, repository='repo', storage_config=storage_config, local_borg_version='1.2.3' + dry_run=False, + repository_path='repo', + storage_config=storage_config, + local_borg_version='1.2.3', ) @@ -138,7 +141,10 @@ def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) module.compact_segments( - dry_run=False, repository='repo', storage_config=storage_config, local_borg_version='1.2.3' + dry_run=False, + repository_path='repo', + storage_config=storage_config, + local_borg_version='1.2.3', ) @@ -148,7 +154,7 @@ def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options( module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={'extra_borg_options': {'compact': '--extra --options'}}, local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 5fb51f3c..42fedd56 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -484,7 +484,7 @@ def test_create_archive_calls_borg_with_parameters(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -527,7 +527,7 @@ def test_create_archive_calls_borg_with_environment(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -572,7 +572,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -617,7 +617,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -660,7 +660,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -700,7 +700,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -744,7 +744,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -784,7 +784,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -827,7 +827,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): module.create_archive( dry_run=True, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -872,7 +872,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete module.create_archive( dry_run=True, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -915,7 +915,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -957,7 +957,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -999,7 +999,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1041,7 +1041,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1088,7 +1088,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1132,7 +1132,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1175,7 +1175,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1223,7 +1223,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1276,7 +1276,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1326,7 +1326,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1380,7 +1380,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1434,7 +1434,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1477,7 +1477,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1520,7 +1520,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1563,7 +1563,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1606,7 +1606,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1648,7 +1648,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1690,7 +1690,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1733,7 +1733,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1777,7 +1777,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1820,7 +1820,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1880,7 +1880,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1943,7 +1943,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2011,7 +2011,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2074,7 +2074,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2115,7 +2115,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): json_output = module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2157,7 +2157,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() json_output = module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2204,7 +2204,7 @@ def test_create_archive_with_source_directories_glob_expands(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo*'], 'repositories': ['repo'], @@ -2247,7 +2247,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo*'], 'repositories': ['repo'], @@ -2289,7 +2289,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo*'], 'repositories': ['repo'], @@ -2331,7 +2331,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2374,7 +2374,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2417,7 +2417,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): module.create_archive( dry_run=False, - repository='{fqdn}', # noqa: FS003 + repository_path='{fqdn}', # noqa: FS003 location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['{fqdn}'], # noqa: FS003 @@ -2459,7 +2459,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2519,7 +2519,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2543,7 +2543,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ with pytest.raises(ValueError): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index a3c03526..92776dd4 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -32,7 +32,7 @@ def test_export_tar_archive_calls_borg_with_path_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=['path1', 'path2'], destination_path='test.tar', @@ -53,7 +53,7 @@ def test_export_tar_archive_calls_borg_with_local_path_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -75,7 +75,7 @@ def test_export_tar_archive_calls_borg_with_remote_path_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -97,7 +97,7 @@ def test_export_tar_archive_calls_borg_with_umask_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -118,7 +118,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -138,7 +138,7 @@ def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -160,7 +160,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -179,7 +179,7 @@ def test_export_tar_archive_calls_borg_with_dry_run_parameter(): module.export_tar_archive( dry_run=True, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -200,7 +200,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -223,7 +223,7 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -245,7 +245,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -265,7 +265,7 @@ def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): module.export_tar_archive( dry_run=False, - repository='server:repo', + repository_path='server:repo', archive='archive', paths=None, destination_path='test.tar', @@ -284,7 +284,7 @@ def test_export_tar_archive_calls_borg_with_stdout_destination_path(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='-', diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index d27026e4..de29cf4d 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -23,7 +23,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_last_archive(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -32,7 +32,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_not_raise(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(('repo',)) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -45,7 +45,7 @@ def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_paramet ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -60,7 +60,7 @@ def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_param ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -74,7 +74,7 @@ def test_extract_last_archive_dry_run_calls_borg_via_local_path(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', - repository='repo', + repository_path='repo', lock_wait=None, local_path='borg1', ) @@ -92,7 +92,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', - repository='repo', + repository_path='repo', lock_wait=None, remote_path='borg1', ) @@ -108,7 +108,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=5 + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=5 ) diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 7bf621da..1985d816 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -50,7 +50,7 @@ def test_make_flags_from_arguments_omits_excludes(): def test_make_repository_flags_with_borg_features_includes_repo_flag(): flexmock(module.feature).should_receive('available').and_return(True) - assert module.make_repository_flags(repository='repo', local_borg_version='1.2.3') == ( + assert module.make_repository_flags(repository_path='repo', local_borg_version='1.2.3') == ( '--repo', 'repo', ) @@ -59,14 +59,16 @@ def test_make_repository_flags_with_borg_features_includes_repo_flag(): def test_make_repository_flags_without_borg_features_includes_omits_flag(): flexmock(module.feature).should_receive('available').and_return(False) - assert module.make_repository_flags(repository='repo', local_borg_version='1.2.3') == ('repo',) + assert module.make_repository_flags(repository_path='repo', local_borg_version='1.2.3') == ( + 'repo', + ) def test_make_repository_archive_flags_with_borg_features_separates_repository_and_archive(): flexmock(module.feature).should_receive('available').and_return(True) assert module.make_repository_archive_flags( - repository='repo', archive='archive', local_borg_version='1.2.3' + repository_path='repo', archive='archive', local_borg_version='1.2.3' ) == ('--repo', 'repo', 'archive',) @@ -74,5 +76,5 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a flexmock(module.feature).should_receive('available').and_return(False) assert module.make_repository_archive_flags( - repository='repo', archive='archive', local_borg_version='1.2.3' + repository_path='repo', archive='archive', local_borg_version='1.2.3' ) == ('repo::archive',) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index ab92065b..fcc556a4 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -23,7 +23,7 @@ def test_display_archives_info_calls_borg_with_parameters(): ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -45,7 +45,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): ) insert_logging_mock(logging.INFO) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -65,7 +65,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu insert_logging_mock(logging.INFO) json_output = module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=True, prefix=None), @@ -90,7 +90,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -110,7 +110,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp insert_logging_mock(logging.DEBUG) json_output = module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=True, prefix=None), @@ -131,7 +131,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): ).and_return('[]') json_output = module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=True, prefix=None), @@ -158,7 +158,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive='archive', json=False, prefix=None), @@ -180,7 +180,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -206,7 +206,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -233,7 +233,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -258,7 +258,7 @@ def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parame ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix='foo'), @@ -284,7 +284,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None, **{argument_name: 'value'}), diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index cedcec84..37af65a3 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -16,7 +16,7 @@ def test_make_list_command_includes_log_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -32,7 +32,7 @@ def test_make_list_command_includes_json_but_not_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), @@ -48,7 +48,7 @@ def test_make_list_command_includes_log_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -64,7 +64,7 @@ def test_make_list_command_includes_json_but_not_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), @@ -79,7 +79,7 @@ def test_make_list_command_includes_json(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), @@ -96,7 +96,7 @@ def test_make_list_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={'lock_wait': 5}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -113,7 +113,7 @@ def test_make_list_command_includes_archive(): ) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=None, json=False), @@ -130,7 +130,7 @@ def test_make_list_command_includes_archive_and_path(): ) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False), @@ -145,7 +145,7 @@ def test_make_list_command_includes_local_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -163,7 +163,7 @@ def test_make_list_command_includes_remote_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -179,7 +179,7 @@ def test_make_list_command_includes_short(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False, short=True), @@ -210,7 +210,7 @@ def test_make_list_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock( @@ -259,7 +259,7 @@ def test_capture_archive_listing_does_not_raise(): flexmock(module).should_receive('make_list_command') module.capture_archive_listing( - repository='repo', + repository_path='repo', archive='archive', storage_config=flexmock(), local_borg_version=flexmock(), @@ -284,7 +284,7 @@ def test_list_archive_calls_borg_with_parameters(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -301,7 +301,7 @@ def test_list_archive_calls_borg_with_parameters(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -318,7 +318,7 @@ def test_list_archive_with_archive_and_json_errors(): with pytest.raises(ValueError): module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -343,7 +343,7 @@ def test_list_archive_calls_borg_with_local_path(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -360,7 +360,7 @@ def test_list_archive_calls_borg_with_local_path(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -408,7 +408,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -433,7 +433,7 @@ def test_list_archive_calls_borg_with_archive(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -450,7 +450,7 @@ def test_list_archive_calls_borg_with_archive(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -480,7 +480,7 @@ def test_list_archive_without_archive_delegates_to_list_repository(): flexmock(module).should_receive('execute_command').never() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -510,7 +510,7 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos flexmock(module).should_receive('execute_command').never() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -537,7 +537,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_fl module.feature.Feature.RLIST, '1.2.3' ).and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -556,7 +556,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_fl ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -586,7 +586,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.rlist).should_receive('make_rlist_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=argparse.Namespace( @@ -601,7 +601,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -619,7 +619,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).and_return(('borg', 'list', '--repo', 'repo', 'archive1')) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -652,7 +652,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 7f2060fa..6161a249 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -20,7 +20,7 @@ def test_mount_archive_calls_borg_with_required_flags(): insert_execute_command_mock(('borg', 'mount', 'repo', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive=None, mount_point='/mnt', paths=None, @@ -39,7 +39,7 @@ def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_a ) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -58,7 +58,7 @@ def test_mount_archive_without_archive_calls_borg_with_repository_flags_only(): insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -77,7 +77,7 @@ def test_mount_archive_calls_borg_with_path_flags(): insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt', 'path1', 'path2')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=['path1', 'path2'], @@ -98,7 +98,7 @@ def test_mount_archive_calls_borg_with_remote_path_flags(): ) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -118,7 +118,7 @@ def test_mount_archive_calls_borg_with_umask_flags(): insert_execute_command_mock(('borg', 'mount', '--umask', '0770', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -137,7 +137,7 @@ def test_mount_archive_calls_borg_with_lock_wait_flags(): insert_execute_command_mock(('borg', 'mount', '--lock-wait', '5', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -157,7 +157,7 @@ def test_mount_archive_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -177,7 +177,7 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_flags(): insert_logging_mock(logging.DEBUG) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -202,7 +202,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): ).once() module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -221,7 +221,7 @@ def test_mount_archive_calls_borg_with_options_flags(): insert_execute_command_mock(('borg', 'mount', '-o', 'super_mount', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index dd240dcb..f33c9933 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -98,7 +98,7 @@ def test_prune_archives_calls_borg_with_parameters(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -114,7 +114,7 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.prune_archives( - repository='repo', + repository_path='repo', storage_config={}, dry_run=False, retention_config=flexmock(), @@ -131,7 +131,7 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.prune_archives( - repository='repo', + repository_path='repo', storage_config={}, dry_run=False, retention_config=flexmock(), @@ -147,7 +147,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) module.prune_archives( - repository='repo', + repository_path='repo', storage_config={}, dry_run=True, retention_config=flexmock(), @@ -164,7 +164,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -181,7 +181,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters( module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -198,7 +198,7 @@ def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_ou module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -215,7 +215,7 @@ def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_out module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -233,7 +233,7 @@ def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', @@ -250,7 +250,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', @@ -266,7 +266,7 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={'extra_borg_options': {'prune': '--extra --options'}}, retention_config=flexmock(), local_borg_version='1.2.3', diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index 612ec11c..e232df22 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -40,7 +40,7 @@ def test_create_repository_calls_borg_with_flags(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -55,7 +55,7 @@ def test_create_repository_with_dry_run_skips_borg_call(): module.create_repository( dry_run=True, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -74,7 +74,7 @@ def test_create_repository_raises_for_borg_rcreate_error(): with pytest.raises(subprocess.CalledProcessError): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -88,7 +88,7 @@ def test_create_repository_skips_creation_when_repository_already_exists(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -103,7 +103,7 @@ def test_create_repository_raises_for_unknown_rinfo_command_error(): with pytest.raises(subprocess.CalledProcessError): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -118,7 +118,7 @@ def test_create_repository_with_source_repository_calls_borg_with_other_repo_fla module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -134,7 +134,7 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -150,7 +150,7 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -166,7 +166,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -182,7 +182,7 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -199,7 +199,7 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -215,7 +215,7 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -230,7 +230,7 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -246,7 +246,7 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -262,7 +262,7 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={'extra_borg_options': {'rcreate': '--extra --options'}}, local_borg_version='2.3.4', encryption_mode='repokey', diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index d9cbf9bc..ec0819cf 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -21,7 +21,7 @@ def test_display_repository_info_calls_borg_with_parameters(): ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -42,7 +42,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -63,7 +63,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): ) insert_logging_mock(logging.INFO) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -82,7 +82,7 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out insert_logging_mock(logging.INFO) json_output = module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), @@ -106,7 +106,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter( insert_logging_mock(logging.DEBUG) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -125,7 +125,7 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou insert_logging_mock(logging.DEBUG) json_output = module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), @@ -145,7 +145,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_parameter(): ).and_return('[]') json_output = module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), @@ -168,7 +168,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -190,7 +190,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_pa ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -213,7 +213,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parame ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 178a820f..a098cafa 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -131,7 +131,7 @@ def test_make_rlist_command_includes_log_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -147,7 +147,7 @@ def test_make_rlist_command_includes_json_but_not_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), @@ -163,7 +163,7 @@ def test_make_rlist_command_includes_log_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -179,7 +179,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), @@ -194,7 +194,7 @@ def test_make_rlist_command_includes_json(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), @@ -211,7 +211,7 @@ def test_make_rlist_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={'lock_wait': 5}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -226,7 +226,7 @@ def test_make_rlist_command_includes_local_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -244,7 +244,7 @@ def test_make_rlist_command_includes_remote_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -262,7 +262,7 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), @@ -277,7 +277,7 @@ def test_make_rlist_command_includes_short(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None, short=True), @@ -307,7 +307,7 @@ def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( @@ -331,7 +331,7 @@ def test_list_repository_calls_borg_with_parameters(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, @@ -347,7 +347,7 @@ def test_list_repository_calls_borg_with_parameters(): ).once() module.list_repository( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, @@ -362,7 +362,7 @@ def test_list_repository_with_json_returns_borg_output(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, @@ -374,7 +374,7 @@ def test_list_repository_with_json_returns_borg_output(): assert ( module.list_repository( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 27b03470..a4814420 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -25,7 +25,7 @@ def test_transfer_archives_calls_borg_with_flags(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -54,7 +54,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): module.transfer_archives( dry_run=True, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -80,7 +80,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): insert_logging_mock(logging.INFO) module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -107,7 +107,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -136,7 +136,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -165,7 +165,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -191,7 +191,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -221,7 +221,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -252,7 +252,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -278,7 +278,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -308,7 +308,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -340,7 +340,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( From b336b9bedfc96532863cf2ace0af0a57a7337af2 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 27 Mar 2023 00:19:23 +0530 Subject: [PATCH 29/94] add tests for repo labels --- tests/unit/config/test_normalize.py | 27 +++++++++++++++++++++++++++ tests/unit/config/test_validate.py | 10 ++++++++++ 2 files changed, 37 insertions(+) diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index fe9e7ac1..1d61a771 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -92,6 +92,21 @@ from borgmatic.config import normalize as module {'location': {'repositories': [{'path': '/repo'}]}}, False, ), + ( + {'location': {'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': 'foo'}]}}, + True, + ), + ( + {'location': {'repositories': [{'path': 'file:///repo', 'label': 'foo'}]}}, + {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, + False, + ), + ( + {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, + {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, + False, + ), ), ) def test_normalize_applies_hard_coded_normalization_to_config( @@ -105,3 +120,15 @@ def test_normalize_applies_hard_coded_normalization_to_config( assert logs else: assert logs == [] + + +def test_normalize_raises_error_if_repository_data_is_not_consistent(): + with pytest.raises(TypeError): + module.normalize( + 'test.yaml', + { + 'location': { + 'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}, 'file:///repo'] + } + }, + ) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 28527226..327a2b44 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -121,6 +121,16 @@ def test_guard_configuration_contains_repository_does_not_raise_when_repository_ ) +def test_guard_configuration_contains_repository_does_not_raise_when_repository_label_in_config(): + + module.guard_configuration_contains_repository( + repository='repo', + configurations={ + 'config.yaml': {'location': {'repositories': [{'path': 'foo/bar', 'label': 'repo'}]}} + }, + ) + + def test_guard_configuration_contains_repository_does_not_raise_when_repository_not_given(): module.guard_configuration_contains_repository( repository=None, configurations={'config.yaml': {'location': {'repositories': ['repo']}}} From acbbd6670a154c79f512894c19f9e6e152544cbd Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Mar 2023 21:26:35 -0700 Subject: [PATCH 30/94] Removing debugging command output. --- borgmatic/execute.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 53d1a098..46dd04f3 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -235,12 +235,10 @@ def execute_command_and_capture_output( env=environment, cwd=working_directory, ) - logger.warning(f'Command output: {output}') except subprocess.CalledProcessError as error: if exit_code_indicates_error(command, error.returncode): raise output = error.output - logger.warning(f'Command output: {output}') return output.decode() if output is not None else None From 8bef1c698b3f2c3cdf2d9c3a44c4bed5a9068282 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 27 Mar 2023 22:16:39 +0530 Subject: [PATCH 31/94] add feature to docs --- docs/how-to/backup-your-databases.md | 3 ++- docs/how-to/extract-a-backup.md | 3 ++- docs/how-to/run-arbitrary-borg-commands.md | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index bc21b659..29bf4772 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -231,7 +231,8 @@ If you have a single repository in your borgmatic configuration file(s), no problem: the `restore` action figures out which repository to use. But if you have multiple repositories configured, then you'll need to specify -the repository path containing the archive to restore. Here's an example: +the repository to use via the `--repository` flag. This can be done either +with the repository's path or its label as configured in your borgmatic configuration file. ```bash borgmatic restore --repository repo.borg --archive host-2023-... diff --git a/docs/how-to/extract-a-backup.md b/docs/how-to/extract-a-backup.md index 4285c784..164fc135 100644 --- a/docs/how-to/extract-a-backup.md +++ b/docs/how-to/extract-a-backup.md @@ -51,7 +51,8 @@ If you have a single repository in your borgmatic configuration file(s), no problem: the `extract` action figures out which repository to use. But if you have multiple repositories configured, then you'll need to specify -the repository path containing the archive to extract. Here's an example: +the repository to use via the `--repository` flag. This can be done either +with the repository's path or its label as configured in your borgmatic configuration file. ```bash borgmatic extract --repository repo.borg --archive host-2023-... diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index 3d119b4c..0777ebba 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -53,7 +53,8 @@ This runs Borg's `rlist` command once on each configured borgmatic repository. (The native `borgmatic rlist` action should be preferred for most use.) What if you only want to run Borg on a single configured borgmatic repository -when you've got several configured? Not a problem. +when you've got several configured? Not a problem. The `--repository` argument +lets you specify the repository to use, either by its path or its label: ```bash borgmatic borg --repository repo.borg break-lock From d0d3a398332b89966e57c3617a6acaa71add85bc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 10:36:39 -0700 Subject: [PATCH 32/94] When a database command errors, display and log the error message instead of swallowing it (#396). --- NEWS | 1 + borgmatic/execute.py | 44 ++++++++++++++++++++++++------- borgmatic/hooks/mysql.py | 6 ++--- tests/integration/test_execute.py | 3 +-- tests/unit/hooks/test_mysql.py | 32 +++++++++++++++------- tests/unit/test_execute.py | 35 ++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 26 deletions(-) diff --git a/NEWS b/NEWS index 7d02c0e2..472eef44 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 1.7.10.dev0 + * #396: When a database command errors, display and log the error message instead of swallowing it. * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" option in borgmatic's location configuration. * #576: Add support for "file://" paths within "repositories" option. diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 46dd04f3..0afa5cca 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -43,6 +43,23 @@ def output_buffer_for_process(process, exclude_stdouts): return process.stderr if process.stdout in exclude_stdouts else process.stdout +def append_last_lines(last_lines, captured_output, line, output_log_level): + ''' + Given a rolling list of last lines, a list of captured output, a line to append, and an output + log level, append the line to the last lines and (if necessary) the captured output. Then log + the line at the requested output log level. + ''' + last_lines.append(line) + + if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT: + last_lines.pop(0) + + if output_log_level is None: + captured_output.append(line) + else: + logger.log(output_log_level, line) + + def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): ''' Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each @@ -98,15 +115,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): # Keep the last few lines of output in case the process errors, and we need the output for # the exception below. - last_lines = buffer_last_lines[ready_buffer] - last_lines.append(line) - if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT: - last_lines.pop(0) - - if output_log_level is None: - captured_outputs[ready_process].append(line) - else: - logger.log(output_log_level, line) + append_last_lines( + buffer_last_lines[ready_buffer], + captured_outputs[ready_process], + line, + output_log_level, + ) if not still_running: break @@ -125,8 +139,18 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): # If an error occurs, include its output in the raised exception so that we don't # inadvertently hide error output. output_buffer = output_buffer_for_process(process, exclude_stdouts) - last_lines = buffer_last_lines[output_buffer] if output_buffer else [] + + # Collect any straggling output lines that came in since we last gathered output. + while output_buffer: # pragma: no cover + line = output_buffer.readline().rstrip().decode() + if not line: + break + + append_last_lines( + last_lines, captured_outputs[process], line, output_log_level=logging.ERROR + ) + if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT: last_lines.insert(0, '...') diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 0bf97745..aeeeee5b 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -88,9 +88,7 @@ def execute_dump_command( + (('--user', database['username']) if 'username' in database else ()) + ('--databases',) + database_names - # Use shell redirection rather than execute_command(output_file=open(...)) to prevent - # the open() call on a named pipe from hanging the main borgmatic process. - + ('>', dump_filename) + + ('--result-file', dump_filename) ) logger.debug( @@ -102,7 +100,7 @@ def execute_dump_command( dump.create_named_pipe_for_dump(dump_filename) return execute_command( - dump_command, shell=True, extra_environment=extra_environment, run_to_completion=False, + dump_command, extra_environment=extra_environment, run_to_completion=False, ) diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 2a9b61d7..09e9093a 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -239,7 +239,6 @@ def test_log_outputs_does_not_error_when_one_process_exits(): def test_log_outputs_truncates_long_error_output(): - flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0 flexmock(module.logger).should_receive('log') flexmock(module).should_receive('command_for_process').and_return('grep') @@ -253,7 +252,7 @@ def test_log_outputs_truncates_long_error_output(): flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout) with pytest.raises(subprocess.CalledProcessError) as error: - module.log_outputs( + flexmock(module, ERROR_OUTPUT_MAX_LINE_COUNT=0).log_outputs( (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg' ) diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 1e8df691..137a88a5 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -149,8 +149,7 @@ def test_execute_dump_command_runs_mysqldump(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--add-drop-database', '--databases', 'foo', '>', 'dump',), - shell=True, + ('mysqldump', '--add-drop-database', '--databases', 'foo', '--result-file', 'dump',), extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -176,8 +175,7 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--databases', 'foo', '>', 'dump',), - shell=True, + ('mysqldump', '--databases', 'foo', '--result-file', 'dump',), extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -214,10 +212,9 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): 'tcp', '--databases', 'foo', - '>', + '--result-file', 'dump', ), - shell=True, extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -243,8 +240,16 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--add-drop-database', '--user', 'root', '--databases', 'foo', '>', 'dump',), - shell=True, + ( + 'mysqldump', + '--add-drop-database', + '--user', + 'root', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment={'MYSQL_PWD': 'trustsome1'}, run_to_completion=False, ).and_return(process).once() @@ -270,8 +275,15 @@ def test_execute_dump_command_runs_mysqldump_with_options(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--stuff=such', '--add-drop-database', '--databases', 'foo', '>', 'dump',), - shell=True, + ( + 'mysqldump', + '--stuff=such', + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment=None, run_to_completion=False, ).and_return(process).once() diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index badcb8fd..14924078 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -65,6 +65,41 @@ def test_output_buffer_for_process_returns_stdout_when_not_excluded(): ) +def test_append_last_lines_under_max_line_count_appends(): + last_lines = ['last'] + flexmock(module.logger).should_receive('log').once() + + module.append_last_lines( + last_lines, captured_output=flexmock(), line='line', output_log_level=flexmock() + ) + + assert last_lines == ['last', 'line'] + + +def test_append_last_lines_over_max_line_count_trims_and_appends(): + original_last_lines = [str(number) for number in range(0, module.ERROR_OUTPUT_MAX_LINE_COUNT)] + last_lines = list(original_last_lines) + flexmock(module.logger).should_receive('log').once() + + module.append_last_lines( + last_lines, captured_output=flexmock(), line='line', output_log_level=flexmock() + ) + + assert last_lines == original_last_lines[1:] + ['line'] + + +def test_append_last_lines_with_output_log_level_none_appends_captured_output(): + last_lines = ['last'] + captured_output = ['captured'] + flexmock(module.logger).should_receive('log').never() + + module.append_last_lines( + last_lines, captured_output=captured_output, line='line', output_log_level=None + ) + + assert captured_output == ['captured', 'line'] + + def test_execute_command_calls_full_command(): full_command = ['foo', 'bar'] flexmock(module.os, environ={'a': 'b'}) From 61c7b8f13cf6a3c905cd82cdc7dac6b5cb8dd421 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 15:54:55 -0700 Subject: [PATCH 33/94] Add optional repository labels so you can select a repository via "--repository yourlabel" at the command-line (#635). --- NEWS | 3 +++ README.md | 7 ++++--- docs/how-to/make-backups-redundant.md | 5 ++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index 472eef44..ed2bafc1 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,9 @@ * #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in borgmatic's storage configuration. * #623: Fix confusing message when an error occurs running actions for a configuration file. + * #635: Add optional repository labels so you can select a repository via "--repository yourlabel" + at the command-line. See the configuration reference for more information: + https://torsion.org/borgmatic/docs/reference/configuration/ * #649: Add documentation on backing up a database running in a container: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers * #655: Fix error when databases are configured and a source directory doesn't exist. diff --git a/README.md b/README.md index 27fc6cd2..eb827ae9 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,10 @@ location: # Paths of local or remote repositories to backup to. repositories: - - ssh://1234@usw-s001.rsync.net/./backups.borg - - ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo - - /var/lib/backups/local.borg + - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo + label: borgbase + - path: /var/lib/backups/local.borg + label: local retention: # Retention policy for how many backups to keep. diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md index f77780fc..71510fc4 100644 --- a/docs/how-to/make-backups-redundant.md +++ b/docs/how-to/make-backups-redundant.md @@ -20,9 +20,8 @@ location: # Paths of local or remote repositories to backup to. repositories: - - ssh://1234@usw-s001.rsync.net/./backups.borg - - ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo - - /var/lib/backups/local.borg + - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo + - path: /var/lib/backups/local.borg ``` When you run borgmatic with this configuration, it invokes Borg once for each From c5ffb76dfa4344ad582e093a25a4488381adb396 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 15:56:49 -0700 Subject: [PATCH 34/94] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index ed2bafc1..9bc9ebf6 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.10.dev0 +1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" option in borgmatic's location configuration. diff --git a/setup.py b/setup.py index 5ea3c2e4..c3682115 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.10.dev0' +VERSION = '1.7.10' setup( From 0e3da7be6341f9b53b74a7bc0f4cac596af9a574 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 16:15:24 -0700 Subject: [PATCH 35/94] Fix repository schema description. --- borgmatic/config/schema.yaml | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 9115eab1..f0dc3b32 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -46,29 +46,20 @@ properties: properties: path: type: string - description: | - Path to local or remote repository - (required). Tildes are expanded. - Multiple repositories are backed up - to in sequence. Borg placeholders - can be used. See the output of - "borg help placeholders" for - details. See ssh_command for SSH - options like identity file or port. - If systemd service is used, then - add local repository paths in the - systemd service file to the - ReadWritePaths list. example: ssh://user@backupserver/./{fqdn} label: type: string - description: | - Optional label for the repository. This - can be used with the --repository flag - to select a repository to backup to. - If not specified, the repository path is - used as the label. example: backupserver + description: | + A required list of local or remote repositories with paths + and optional labels (which can be used with the --repository + flag to select a repository). Tildes are expanded. Multiple + repositories are backed up to in sequence. Borg placeholders + can be used. See the output of "borg help placeholders" for + details. See ssh_command for SSH options like identity file + or port. If systemd service is used, then add local + repository paths in the systemd service file to the + ReadWritePaths list. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver From 489ae080e525c5b9c735124d269c3de6f6f43c58 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 21:49:31 -0700 Subject: [PATCH 36/94] Update docs with a few more "path:" repositories references (#635). --- ...p-to-a-removable-drive-or-an-intermittent-server.md | 10 ++++++++-- docs/how-to/make-backups-redundant.md | 3 +++ docs/how-to/upgrade.md | 7 +++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index 0f578551..098b03f6 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -49,9 +49,12 @@ location: - /home repositories: - - /mnt/removable/backup.borg + - path: /mnt/removable/backup.borg ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + Then, write a `before_backup` hook in that same configuration file that uses the external `findmnt` utility to see whether the drive is mounted before proceeding. @@ -79,13 +82,16 @@ location: - /home repositories: - - ssh://me@buddys-server.org/./backup.borg + - path: ssh://me@buddys-server.org/./backup.borg hooks: before_backup: - ping -q -c 1 buddys-server.org > /dev/null || exit 75 ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + Or to only run backups if the battery level is high enough: ```yaml diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md index 71510fc4..a7e21028 100644 --- a/docs/how-to/make-backups-redundant.md +++ b/docs/how-to/make-backups-redundant.md @@ -24,6 +24,9 @@ location: - path: /var/lib/backups/local.borg ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + When you run borgmatic with this configuration, it invokes Borg once for each configured repository in sequence. (So, not in parallel.) That means—in each repository—borgmatic creates a single new backup archive containing all of diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index 296b3f88..69b5f5b8 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -145,15 +145,18 @@ like this: ```yaml location: repositories: - - original.borg + - path: original.borg ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + Change it to a new (not yet created) repository path: ```yaml location: repositories: - - upgraded.borg + - path: upgraded.borg ``` Then, run the `rcreate` action (formerly `init`) to create that new Borg 2 From 7a8e0e89ddcbfcc36722df102bd29c3c5ff49b7e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 21:54:01 -0700 Subject: [PATCH 37/94] Mention prior versions of borgmatic in repositories schema. --- borgmatic/config/schema.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index f0dc3b32..3e94b8d2 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -59,7 +59,8 @@ properties: details. See ssh_command for SSH options like identity file or port. If systemd service is used, then add local repository paths in the systemd service file to the - ReadWritePaths list. + ReadWritePaths list. Prior to borgmatic 1.7.10, repositories + was just a list of plain path strings. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver From 1bcdebd1cc6731142cbbcee04d551fbb09480c8a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 23:16:44 -0700 Subject: [PATCH 38/94] Fix multiple repositories example. --- docs/how-to/make-backups-redundant.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md index a7e21028..2a4b8121 100644 --- a/docs/how-to/make-backups-redundant.md +++ b/docs/how-to/make-backups-redundant.md @@ -34,9 +34,8 @@ your source directories. Here's a way of visualizing what borgmatic does with the above configuration: -1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg` -2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo` -3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg` +1. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo` +2. Backup `/home` and `/etc` to `/var/lib/backups/local.borg` This gives you redundancy of your data across repositories and even potentially across providers. From dcefded0fa5fd88ceddf48e62826d3ef99351c0d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 23:21:14 -0700 Subject: [PATCH 39/94] Document that most command-line flags are not config-file-able (#461). --- docs/reference/command-line.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/reference/command-line.md b/docs/reference/command-line.md index 8997cd93..d1b67dc5 100644 --- a/docs/reference/command-line.md +++ b/docs/reference/command-line.md @@ -7,8 +7,10 @@ eleventyNavigation: --- ## borgmatic options -Here are all of the available borgmatic command-line options. This includes the separate options for -each action sub-command: +Here are all of the available borgmatic command-line options, including the +separate options for each action sub-command. Note that most of the +flags listed here do have equivalents in borgmatic's [configuration +file](https://torsion.org/borgmatic/docs/reference/configuration/). ``` {% include borgmatic/command-line.txt %} From 67a349ae4418718e73cf0b90753ba868e18f7735 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 23:28:36 -0700 Subject: [PATCH 40/94] I had one job... (#461). --- docs/reference/command-line.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/command-line.md b/docs/reference/command-line.md index d1b67dc5..9cfdd7ee 100644 --- a/docs/reference/command-line.md +++ b/docs/reference/command-line.md @@ -9,7 +9,7 @@ eleventyNavigation: Here are all of the available borgmatic command-line options, including the separate options for each action sub-command. Note that most of the -flags listed here do have equivalents in borgmatic's [configuration +flags listed here do not have equivalents in borgmatic's [configuration file](https://torsion.org/borgmatic/docs/reference/configuration/). ``` From d96f2239c1a50c9376b50fbde570a9b2074da9f0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 23:43:39 -0700 Subject: [PATCH 41/94] Update OpenBSD borgmatic link. --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 52962c34..2ab1e8d5 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -90,7 +90,7 @@ installing borgmatic: * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) - * [OpenBSD](http://ports.su/sysutils/borgmatic) + * [OpenBSD](https://openports.pl/path/sysutils/borgmatic) * [openSUSE](https://software.opensuse.org/package/borgmatic) * [macOS (via Homebrew)](https://formulae.brew.sh/formula/borgmatic) * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/) From 2d08a63e603d53ec2a949405cdccbbf809bcad37 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 28 Mar 2023 22:14:50 +0530 Subject: [PATCH 42/94] fix: make check repositories work with dict and str repositories --- borgmatic/config/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index abcfe3d2..96025b3a 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -69,7 +69,7 @@ def apply_logical_validation(config_filename, parsed_configuration): location_repositories = parsed_configuration.get('location', {}).get('repositories') check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) for repository in check_repositories: - if repository not in location_repositories: + if not any(repositories_match(repository, config_repository) for config_repository in location_repositories): raise Validation_error( config_filename, ( From ce22d2d30252a23ff8aea0c9afb0491d618aca75 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 28 Mar 2023 22:29:21 +0530 Subject: [PATCH 43/94] reformat --- borgmatic/config/validate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 96025b3a..fcf29d38 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -69,7 +69,10 @@ def apply_logical_validation(config_filename, parsed_configuration): location_repositories = parsed_configuration.get('location', {}).get('repositories') check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) for repository in check_repositories: - if not any(repositories_match(repository, config_repository) for config_repository in location_repositories): + if not any( + repositories_match(repository, config_repository) + for config_repository in location_repositories + ): raise Validation_error( config_filename, ( From 08e358e27f09b84f08e932973ddf666356c889d4 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 28 Mar 2023 22:51:35 +0530 Subject: [PATCH 44/94] add and update tests --- tests/unit/config/test_validate.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 327a2b44..d5606bde 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -51,17 +51,33 @@ def test_apply_locical_validation_raises_if_unknown_repository_in_check_reposito ) -def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_repositories(): +def test_apply_locical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { - 'location': {'repositories': ['repo.borg', 'other.borg']}, + 'location': {'repositories': [{'path': 'repo.borg'}, {'path': 'other.borg'}]}, 'retention': {'keep_secondly': 1000}, 'consistency': {'check_repositories': ['repo.borg']}, }, ) +def test_apply_locical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): + module.apply_logical_validation( + 'config.yaml', + { + 'location': { + 'repositories': [ + {'path': 'repo.borg', 'label': 'my_repo'}, + {'path': 'other.borg', 'label': 'other_repo'}, + ] + }, + 'retention': {'keep_secondly': 1000}, + 'consistency': {'check_repositories': ['my_repo']}, + }, + ) + + def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present(): module.apply_logical_validation( 'config.yaml', From 59fe01b56d18e92ee49f6b8417c8f867a3f904df Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 11:09:25 -0700 Subject: [PATCH 45/94] Update script comment. --- scripts/run-end-to-end-dev-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run-end-to-end-dev-tests b/scripts/run-end-to-end-dev-tests index 3ee37acc..575de00d 100755 --- a/scripts/run-end-to-end-dev-tests +++ b/scripts/run-end-to-end-dev-tests @@ -1,7 +1,7 @@ #!/bin/sh -# This script is for running all tests, including end-to-end tests, on a developer machine. It sets -# up database containers to run tests against, runs the tests, and then tears down the containers. +# This script is for running end-to-end tests on a developer machine. It sets up database containers +# to run tests against, runs the tests, and then tears down the containers. # # Run this script from the root directory of the borgmatic source. # From 3512191f3e8f3c4ac537746dc35f34631e7a19ba Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 11:45:55 -0700 Subject: [PATCH 46/94] Add check_repositories regression fix to NEWS (#662). --- NEWS | 3 +++ setup.py | 2 +- tests/unit/config/test_validate.py | 6 +++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 9bc9ebf6..7f2353d6 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.7.11.dev0 + * #662: Fix regression in which "check_repositories" option failed to match repositories. + 1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" diff --git a/setup.py b/setup.py index c3682115..bc4e4b75 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.10' +VERSION = '1.7.11.dev0' setup( diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index d5606bde..11f3127c 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -37,7 +37,7 @@ def test_validation_error_string_contains_errors(): assert 'uh oh' in result -def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories(): +def test_apply_logical_validation_raises_if_unknown_repository_in_check_repositories(): flexmock(module).format_json_error = lambda error: error.message with pytest.raises(module.Validation_error): @@ -51,7 +51,7 @@ def test_apply_locical_validation_raises_if_unknown_repository_in_check_reposito ) -def test_apply_locical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): +def test_apply_logical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { @@ -62,7 +62,7 @@ def test_apply_locical_validation_does_not_raise_if_known_repository_path_in_che ) -def test_apply_locical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): +def test_apply_logical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { From f709125110eb4c58204ed19b3ee591497523d36d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 12:02:07 -0700 Subject: [PATCH 47/94] Error out if run-full-tests is run not inside a test container. --- .drone.yml | 2 ++ scripts/run-full-tests | 9 ++++++++- tests/end-to-end/docker-compose.yaml | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index cf4c358a..354cff95 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,6 +24,8 @@ clone: steps: - name: build image: alpine:3.13 + environment: + TEST_CONTAINER: true pull: always commands: - scripts/run-full-tests diff --git a/scripts/run-full-tests b/scripts/run-full-tests index cbd824ce..bf26c212 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -8,7 +8,14 @@ # For more information, see: # https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/ -set -ex +set -e + +if [ -z "$TEST_CONTAINER" ] ; then + echo "This script is designed to work inside a test container and is not intended to" + echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute" + echo "scripts/run-end-to-end-dev-tests instead." + exit 1 +fi apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \ py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 090cf12c..884edba0 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -17,6 +17,8 @@ services: MONGO_INITDB_ROOT_PASSWORD: test tests: image: alpine:3.13 + environment: + TEST_CONTAINER: true volumes: - "../..:/app:ro" tmpfs: From aaf3462d1704a39149708180ab9bf2db9b145244 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 12:03:12 -0700 Subject: [PATCH 48/94] Fix Drone intentation. --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 354cff95..9a7630ac 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,8 +24,8 @@ clone: steps: - name: build image: alpine:3.13 - environment: - TEST_CONTAINER: true + environment: + TEST_CONTAINER: true pull: always commands: - scripts/run-full-tests From 010b82d6d8302616c8fa34973d795afa992f6315 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 12:45:39 -0700 Subject: [PATCH 49/94] Remove unnecessary cd in dev documentation. --- docs/how-to/develop-on-borgmatic.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index d4315457..796c7367 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -25,7 +25,7 @@ so that you can run borgmatic commands while you're hacking on them to make sure your changes work. ```bash -cd borgmatic/ +cd borgmatic pip3 install --user --editable . ``` @@ -51,7 +51,6 @@ pip3 install --user tox Finally, to actually run tests, run: ```bash -cd borgmatic tox ``` From fc2c181b740b516fab6fb9e9e0907aea5707cc82 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 15:31:37 -0700 Subject: [PATCH 50/94] Add missing Docker Compose depends. --- tests/end-to-end/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 884edba0..80f12e9a 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -30,3 +30,4 @@ services: depends_on: - postgresql - mysql + - mongodb From b27e625a7772860fb5ab2042c87bcc74b98556d8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 15:44:38 -0700 Subject: [PATCH 51/94] Update schema comment for check_repositories to mention labels (#635). --- borgmatic/config/schema.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3e94b8d2..469b1f52 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -538,12 +538,12 @@ properties: items: type: string description: | - Paths to a subset of the repositories in the location - section on which to run consistency checks. Handy in case - some of your repositories are very large, and so running - consistency checks on them would take too long. Defaults to - running consistency checks on all repositories configured in - the location section. + Paths or labels for a subset of the repositories in the + location section on which to run consistency checks. Handy + in case some of your repositories are very large, and so + running consistency checks on them would take too long. + Defaults to running consistency checks on all repositories + configured in the location section. example: - user@backupserver:sourcehostname.borg check_last: From 5f595f7ac3fbfc8c19694baaa8b08e613b60a98b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 30 Mar 2023 23:21:20 -0700 Subject: [PATCH 52/94] Fix regression in which the "transfer" action produced a traceback (#663). --- NEWS | 1 + borgmatic/actions/transfer.py | 4 ++-- tests/unit/actions/test_transfer.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 7f2353d6..8ffbbde3 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.7.11.dev0 * #662: Fix regression in which "check_repositories" option failed to match repositories. + * #663: Fix regression in which the "transfer" action produced a traceback. 1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 628f2735..8089fd4e 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,10 +17,10 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' - logger.info(f'{repository}: Transferring archives to repository') + logger.info(f'{repository["path"]}: Transferring archives to repository') borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, transfer_arguments, diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py index cc9f1386..58d8a160 100644 --- a/tests/unit/actions/test_transfer.py +++ b/tests/unit/actions/test_transfer.py @@ -10,7 +10,7 @@ def test_run_transfer_does_not_raise(): global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_transfer( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, transfer_arguments=transfer_arguments, From 3f78ac4085f689839d9f84d5328cd34b53333e29 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 31 Mar 2023 15:21:08 -0700 Subject: [PATCH 53/94] Automatically use the "archive_name_format" option to filter which archives get used for borgmatic actions that operate on multiple archives (#479). --- NEWS | 5 ++ borgmatic/borg/check.py | 29 +++--- borgmatic/borg/flags.py | 18 ++++ borgmatic/borg/info.py | 6 +- borgmatic/borg/prune.py | 22 ++--- borgmatic/borg/rlist.py | 6 +- borgmatic/borg/transfer.py | 13 ++- borgmatic/config/schema.yaml | 29 +++--- docs/how-to/make-per-application-backups.md | 90 +++++++++++++++---- tests/unit/borg/test_check.py | 99 +++++++++++++-------- tests/unit/borg/test_flags.py | 33 +++++++ tests/unit/borg/test_info.py | 91 ++++++++++++++++++- tests/unit/borg/test_prune.py | 61 +++++++------ tests/unit/borg/test_rlist.py | 69 ++++++++++++++ tests/unit/borg/test_transfer.py | 45 +++++++++- 15 files changed, 488 insertions(+), 128 deletions(-) diff --git a/NEWS b/NEWS index 8ffbbde3..f0518876 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,9 @@ 1.7.11.dev0 + * #479: Automatically use the "archive_name_format" option to filter which archives get used for + borgmatic actions that operate on multiple archives. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming + * #479: The "prefix" options have been deprecated in favor of the new "archive_name_format" + auto-matching behavior (see above). * #662: Fix regression in which "check_repositories" option failed to match repositories. * #663: Fix regression in which the "transfer" action produced a traceback. diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 993a3c0a..4630782a 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -12,7 +12,6 @@ DEFAULT_CHECKS = ( {'name': 'repository', 'frequency': '1 month'}, {'name': 'archives', 'frequency': '1 month'}, ) -DEFAULT_PREFIX = '{hostname}-' # noqa: FS003 logger = logging.getLogger(__name__) @@ -146,9 +145,10 @@ def filter_checks_on_frequency( return tuple(filtered_checks) -def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): +def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None): ''' - Given the local Borg version and a parsed sequence of checks, transform the checks into tuple of + Given the local Borg version, a storge configuration dict, a parsed sequence of checks, the + check last value, and a consistency check prefix, transform the checks into tuple of command-line flags. For example, given parsed checks of: @@ -174,10 +174,19 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): if 'archives' in checks: last_flags = ('--last', str(check_last)) if check_last else () - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): - match_archives_flags = ('--match-archives', f'sh:{prefix}*') if prefix else () - else: - match_archives_flags = ('--glob-archives', f'{prefix}*') if prefix else () + match_archives_flags = ( + ( + ('--match-archives', f'sh:{prefix}*') + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) + else ('--glob-archives', f'{prefix}*') + ) + if prefix + else ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) + ) else: last_flags = () match_archives_flags = () @@ -291,7 +300,7 @@ def check_archives( extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') if set(checks).intersection({'repository', 'archives', 'data'}): - lock_wait = storage_config.get('lock_wait', None) + lock_wait = storage_config.get('lock_wait') verbosity_flags = () if logger.isEnabledFor(logging.INFO): @@ -299,12 +308,12 @@ def check_archives( if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') - prefix = consistency_config.get('prefix', DEFAULT_PREFIX) + prefix = consistency_config.get('prefix') full_command = ( (local_path, 'check') + (('--repair',) if repair else ()) - + make_check_flags(local_borg_version, checks, check_last, prefix) + + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix) + (('--remote-path', remote_path) if remote_path else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 5dcebf50..1354d658 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -1,4 +1,5 @@ import itertools +import re from borgmatic.borg import feature @@ -56,3 +57,20 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version): if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) else (f'{repository_path}::{archive}',) ) + + +def make_match_archives_flags(archive_name_format, local_borg_version): + ''' + Return the match archives flags that would match archives created with the given archive name + format (if any). This is done by replacing certain archive name format placeholders for + ephemeral data (like "{now}") with globs. + ''' + if not archive_name_format: + return () + + match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format) + + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): + return ('--match-archives', f'sh:{match_archives}') + else: + return ('--glob-archives', f'{match_archives}') diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index 6142104f..fa59165c 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -44,7 +44,11 @@ def display_archives_info( else flags.make_flags('glob-archives', f'{info_arguments.prefix}*') ) if info_arguments.prefix - else () + else ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) ) + flags.make_flags_from_arguments( info_arguments, excludes=('repository', 'archive', 'prefix') diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index d21ceee3..08cb0c17 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -7,10 +7,10 @@ from borgmatic.execute import execute_command logger = logging.getLogger(__name__) -def make_prune_flags(retention_config, local_borg_version): +def make_prune_flags(storage_config, retention_config, local_borg_version): ''' - Given a retention config dict mapping from option name to value, tranform it into an iterable of - command-line name-value flag pairs. + Given a retention config dict mapping from option name to value, tranform it into an sequence of + command-line flags. For example, given a retention config of: @@ -24,7 +24,7 @@ def make_prune_flags(retention_config, local_borg_version): ) ''' config = retention_config.copy() - prefix = config.pop('prefix', '{hostname}-') # noqa: FS003 + prefix = config.pop('prefix', None) if prefix: if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): @@ -32,10 +32,16 @@ def make_prune_flags(retention_config, local_borg_version): else: config['glob_archives'] = f'{prefix}*' - return ( + flag_pairs = ( ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items() ) + return tuple( + element for pair in flag_pairs for element in pair + ) + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + def prune_archives( dry_run, @@ -60,11 +66,7 @@ def prune_archives( full_command = ( (local_path, 'prune') - + tuple( - element - for pair in make_prune_flags(retention_config, local_borg_version) - for element in pair - ) + + make_prune_flags(storage_config, retention_config, local_borg_version) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 8625363b..f3935a16 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -94,7 +94,11 @@ def make_rlist_command( else flags.make_flags('glob-archives', f'{rlist_arguments.prefix}*') ) if rlist_arguments.prefix - else () + else ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) ) + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 29e205c7..a350eff7 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -34,9 +34,16 @@ def transfer_archives( 'match-archives', transfer_arguments.match_archives or transfer_arguments.archive ) ) - + flags.make_flags_from_arguments( - transfer_arguments, - excludes=('repository', 'source_repository', 'archive', 'match_archives'), + + ( + flags.make_flags_from_arguments( + transfer_arguments, + excludes=('repository', 'source_repository', 'archive', 'match_archives'), + ) + or ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) ) + flags.make_repository_flags(repository_path, local_borg_version) + flags.make_flags('other-repo', transfer_arguments.source_repository) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 469b1f52..d4374525 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -378,11 +378,9 @@ properties: description: | Name of the archive. Borg placeholders can be used. See the output of "borg help placeholders" for details. Defaults to - "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this - option, consider also specifying a prefix in the retention - and consistency sections to avoid accidental - pruning/checking of archives with different archive name - formats. + "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". When running + actions like rlist, info, or check, borgmatic automatically + tries to match only archives created with this name format. example: "{hostname}-documents-{now}" relocated_repo_access_is_ok: type: boolean @@ -477,10 +475,12 @@ properties: prefix: type: string description: | - When pruning, only consider archive names starting with this - prefix. Borg placeholders can be used. See the output of - "borg help placeholders" for details. Defaults to - "{hostname}-". Use an empty value to disable the default. + Deprecated. When pruning, only consider archive names + starting with this prefix. Borg placeholders can be used. + See the output of "borg help placeholders" for details. + If a prefix is not specified, borgmatic defaults to + matching archives based on the archive_name_format (see + above). example: sourcehostname consistency: type: object @@ -556,11 +556,12 @@ properties: prefix: type: string description: | - When performing the "archives" check, only consider archive - names starting with this prefix. Borg placeholders can be - used. See the output of "borg help placeholders" for - details. Defaults to "{hostname}-". Use an empty value to - disable the default. + Deprecated. When performing the "archives" check, only + consider archive names starting with this prefix. Borg + placeholders can be used. See the output of "borg help + placeholders" for details. If a prefix is not specified, + borgmatic defaults to matching archives based on the + archive_name_format (see above). example: sourcehostname output: type: object diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index e5ba037f..6e5d999a 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -54,6 +54,72 @@ choice](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot), each entry using borgmatic's `--config` flag instead of relying on `/etc/borgmatic.d`. + +## Archive naming + +If you've got multiple borgmatic configuration files, you might want to create +archives with different naming schemes for each one. This is especially handy +if each configuration file is backing up to the same Borg repository but you +still want to be able to distinguish backup archives for one application from +another. + +borgmatic supports this use case with an `archive_name_format` option. The +idea is that you define a string format containing a number of [Borg +placeholders](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-placeholders), +and borgmatic uses that format to name any new archive it creates. For +instance: + +```yaml +location: + ... + archive_name_format: home-directories-{now} +``` + +This means that when borgmatic creates an archive, its name will start with +the string `home-directories-` and end with a timestamp for its creation time. +If `archive_name_format` is unspecified, the default is +`{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a +timestamp in a particular format. + +New in version 1.7.11 borgmatic +uses the `archive_name_format` option to automatically limit which archives +get used for actions operating on multiple archives. This prevents, for +instance, duplicate archives from showing up in `rlist` or `info` results—even +if the same repository appears in multiple borgmatic configuration files. To +take advantage of this feature, simply use a different `archive_name_format` +in each configuration file. + +Under the hood, borgmatic accomplishes this by substituting globs for certain +ephemeral data placeholders in your `archive_name_format`—and using the result +to filter archives for supported actions. + +For instance, let's say that you have this in your configuration: + +```yaml +location: + ... + archive_name_format: {hostname}-user-data-{now} +``` + +borgmatic considers `{now}` an emphemeral placeholder that will probably +change per archive, while `{hostname}` won't. So it turns the example value +into `{hostname}-user-data-*` and applies it to filter down the set of +archives used for actions like `rlist`, `info`, `prune`, `check`, etc. + +The end result is that when borgmatic runs the actions for a particular +application-specific configuration file, it only operates on the archives +created for that application. Of course, this doesn't apply to actions like +`compact` that operate on an entire repository. + +Prior to 1.7.11 The way to +limit the archives used was a `prefix` option in the `retention` section for +matching against the start of archive names used for a `prune` action and +a separate `prefix` option in the `consistency` section for matching against +the start of archive names used for a `check` action. Both of these options +are deprecated in favor of the auto-matching behavior in newer versions of +borgmatic mentioned above. + + ## Configuration includes Once you have multiple different configuration files, you might want to share @@ -272,7 +338,7 @@ Here's an example usage: ```yaml constants: user: foo - my_prefix: bar- + archive_prefix: bar location: source_directories: @@ -281,20 +347,14 @@ location: ... storage: - archive_name_format: '{my_prefix}{now}' - -retention: - prefix: {my_prefix} - -consistency: - prefix: {my_prefix} + archive_name_format: '{archive_prefix}-{now}' ``` In this example, when borgmatic runs, all instances of `{user}` get replaced -with `foo` and all instances of `{my_prefix}` get replaced with `bar-`. (And -in this particular example, `{now}` doesn't get replaced with anything, but -gets passed directly to Borg.) After substitution, the logical result looks -something like this: +with `foo` and all instances of `{archive-prefix}` get replaced with `bar-`. +(And in this particular example, `{now}` doesn't get replaced with anything, +but gets passed directly to Borg.) After substitution, the logical result +looks something like this: ```yaml location: @@ -305,12 +365,6 @@ location: storage: archive_name_format: 'bar-{now}' - -retention: - prefix: bar- - -consistency: - prefix: bar- ``` An alternate to constants is passing in your values via [environment diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 75755565..7c233bcf 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -189,150 +189,170 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_ def test_make_check_flags_with_repository_check_returns_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',)) + flags = module.make_check_flags('1.2.3', {}, ('repository',)) assert flags == ('--repository-only',) def test_make_check_flags_with_archives_check_returns_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',)) + flags = module.make_check_flags('1.2.3', {}, ('archives',)) assert flags == ('--archives-only',) def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',)) + flags = module.make_check_flags('1.2.3', {}, ('data',)) assert flags == ('--archives-only', '--verify-data',) def test_make_check_flags_with_extract_omits_extract_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('extract',)) + flags = module.make_check_flags('1.2.3', {}, ('extract',)) assert flags == () def test_make_check_flags_with_repository_and_data_checks_does_not_return_repository_only(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'data',)) + flags = module.make_check_flags('1.2.3', {}, ('repository', 'data',)) assert flags == ('--verify-data',) -def test_make_check_flags_with_default_checks_and_default_prefix_returns_default_flags(): +def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo',) + + assert flags == ('--match-archives', 'sh:foo*') + + +def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', ('repository', 'archives'), prefix=module.DEFAULT_PREFIX + '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', ) - assert flags == ('--match-archives', f'sh:{module.DEFAULT_PREFIX}*') + assert flags == ('--match-archives', 'sh:foo*') -def test_make_check_flags_with_all_checks_and_default_prefix_returns_default_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - - flags = module.make_check_flags( - '1.2.3', ('repository', 'archives', 'extract'), prefix=module.DEFAULT_PREFIX - ) - - assert flags == ('--match-archives', f'sh:{module.DEFAULT_PREFIX}*') - - -def test_make_check_flags_with_all_checks_and_default_prefix_without_borg_features_returns_glob_archives_flags(): +def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', ('repository', 'archives', 'extract'), prefix=module.DEFAULT_PREFIX + '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', ) - assert flags == ('--glob-archives', f'{module.DEFAULT_PREFIX}*') + assert flags == ('--glob-archives', 'foo*') def test_make_check_flags_with_archives_check_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('archives',), check_last=3) assert flags == ('--archives-only', '--last', '3') def test_make_check_flags_with_data_check_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('data',), check_last=3) assert flags == ('--archives-only', '--last', '3', '--verify-data') def test_make_check_flags_with_repository_check_and_last_omits_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('repository',), check_last=3) assert flags == ('--repository-only',) def test_make_check_flags_with_default_checks_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'archives'), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) assert flags == ('--last', '3') def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix='foo-') assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('data',), prefix='foo-') assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data') -def test_make_check_flags_with_archives_check_and_empty_prefix_omits_match_archives_flag(): +def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) - flags = module.make_check_flags('1.2.3', ('archives',), prefix='') + flags = module.make_check_flags( + '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 + ) - assert flags == ('--archives-only',) + assert flags == ('--archives-only', '--match-archives', 'sh:bar-*') def test_make_check_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), prefix=None) + flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix=None) assert flags == ('--archives-only',) def test_make_check_flags_with_repository_check_and_prefix_omits_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('repository',), prefix='foo-') assert flags == ('--repository-only',) def test_make_check_flags_with_default_checks_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'archives'), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') assert flags == ('--match-archives', 'sh:foo-*') @@ -427,7 +447,7 @@ def test_check_archives_calls_borg_with_parameters(checks): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', {}, checks, check_last, prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) @@ -581,7 +601,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', {}, checks, check_last, prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1', 'check', 'repo')) @@ -608,7 +628,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', {}, checks, check_last, prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo')) @@ -628,6 +648,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): checks = ('repository',) check_last = flexmock() + storage_config = {'lock_wait': 5} consistency_config = {'check_last': check_last} flexmock(module).should_receive('parse_checks') flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) @@ -635,7 +656,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', storage_config, checks, check_last, None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo')) @@ -645,7 +666,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.check_archives( repository_path='repo', location_config={}, - storage_config={'lock_wait': 5}, + storage_config=storage_config, consistency_config=consistency_config, local_borg_version='1.2.3', ) @@ -662,7 +683,7 @@ def test_check_archives_with_retention_prefix(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, prefix + '1.2.3', {}, checks, check_last, prefix ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 1985d816..cf9eedbb 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -1,3 +1,4 @@ +import pytest from flexmock import flexmock from borgmatic.borg import flags as module @@ -78,3 +79,35 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a assert module.make_repository_archive_flags( repository_path='repo', archive='archive', local_borg_version='1.2.3' ) == ('repo::archive',) + + +@pytest.mark.parametrize( + 'archive_name_format,feature_available,expected_result', + ( + (None, True, ()), + ('', True, ()), + ( + '{hostname}-docs-{now}', # noqa: FS003 + True, + ('--match-archives', 'sh:{hostname}-docs-*'), # noqa: FS003 + ), + ('{utcnow}-docs-{user}', True, ('--match-archives', 'sh:*-docs-{user}')), # noqa: FS003 + ('{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003 + ( + 'stuff-{now:%Y-%m-%dT%H:%M:%S.%f}', # noqa: FS003 + True, + ('--match-archives', 'sh:stuff-*'), + ), + ('{hostname}-docs-{now}', False, ('--glob-archives', '{hostname}-docs-*')), # noqa: FS003 + ('{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003 + ), +) +def test_make_match_archives_flags_makes_flags_with_globs( + archive_name_format, feature_available, expected_result +): + flexmock(module.feature).should_receive('available').and_return(feature_available) + + assert ( + module.make_match_archives_flags(archive_name_format, local_borg_version=flexmock()) + == expected_result + ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index fcc556a4..b25f29f6 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -12,6 +12,9 @@ def test_display_archives_info_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -34,6 +37,9 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -56,6 +62,9 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -78,6 +87,9 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -101,6 +113,9 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -123,6 +138,9 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -147,6 +165,9 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'archive' ).and_return(('--match-archives', 'archive')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -169,6 +190,9 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -195,6 +219,9 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para flexmock(module.flags).should_receive('make_flags').with_args( 'remote-path', 'borg1' ).and_return(('--remote-path', 'borg1')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -221,6 +248,9 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete flexmock(module.flags).should_receive('make_flags').with_args('lock-wait', 5).and_return( ('--lock-wait', '5') ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) storage_config = {'lock_wait': 5} @@ -240,13 +270,16 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete ) -def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parameters(): +def test_display_archives_info_transforms_prefix_into_match_archives_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -265,12 +298,68 @@ def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parame ) +def test_display_archives_info_prefers_prefix_over_archive_name_format(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args( + 'match-archives', 'sh:foo*' + ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix='foo'), + ) + + +def test_display_archives_info_transforms_archive_name_format_into_match_archives_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:bar-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None), + ) + + @pytest.mark.parametrize('argument_name', ('match_archives', 'sort_by', 'first', 'last')) def test_display_archives_info_passes_through_arguments_to_borg(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index f33c9933..1c8843e2 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -18,18 +18,17 @@ def insert_execute_command_mock(prune_command, output_log_level): ).once() -BASE_PRUNE_FLAGS = (('--keep-daily', '1'), ('--keep-weekly', '2'), ('--keep-monthly', '3')) +BASE_PRUNE_FLAGS = ('--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') -def test_make_prune_flags_returns_flags_from_config_plus_default_prefix_glob(): +def test_make_prune_flags_returns_flags_from_config(): retention_config = OrderedDict((('keep_daily', 1), ('keep_weekly', 2), ('keep_monthly', 3))) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') - assert tuple(result) == BASE_PRUNE_FLAGS + ( - ('--match-archives', 'sh:{hostname}-*'), # noqa: FS003 - ) + assert result == BASE_PRUNE_FLAGS def test_make_prune_flags_accepts_prefix_with_placeholders(): @@ -37,15 +36,18 @@ def test_make_prune_flags_accepts_prefix_with_placeholders(): (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 ) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') expected = ( - ('--keep-daily', '1'), - ('--match-archives', 'sh:Documents_{hostname}-{now}*'), # noqa: FS003 + '--keep-daily', + '1', + '--match-archives', + 'sh:Documents_{hostname}-{now}*', # noqa: FS003 ) - assert tuple(result) == expected + assert result == expected def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives(): @@ -53,37 +55,38 @@ def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives() (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 ) flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') expected = ( - ('--keep-daily', '1'), - ('--glob-archives', 'Documents_{hostname}-{now}*'), # noqa: FS003 + '--keep-daily', + '1', + '--glob-archives', + 'Documents_{hostname}-{now}*', # noqa: FS003 ) - assert tuple(result) == expected + assert result == expected -def test_make_prune_flags_treats_empty_prefix_as_no_prefix(): - retention_config = OrderedDict((('keep_daily', 1), ('prefix', ''))) - flexmock(module.feature).should_receive('available').and_return(True) - - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') - - expected = (('--keep-daily', '1'),) - - assert tuple(result) == expected - - -def test_make_prune_flags_treats_none_prefix_as_no_prefix(): +def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): + storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 retention_config = OrderedDict((('keep_daily', 1), ('prefix', None))) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') - expected = (('--keep-daily', '1'),) + expected = ( + '--keep-daily', + '1', + '--match-archives', + 'sh:bar-*', # noqa: FS003 + ) - assert tuple(result) == expected + assert result == expected PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index a098cafa..8a10b075 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -127,6 +127,9 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter def test_make_rlist_command_includes_log_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -143,6 +146,9 @@ def test_make_rlist_command_includes_log_info(): def test_make_rlist_command_includes_json_but_not_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -159,6 +165,9 @@ def test_make_rlist_command_includes_json_but_not_info(): def test_make_rlist_command_includes_log_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -175,6 +184,9 @@ def test_make_rlist_command_includes_log_debug(): def test_make_rlist_command_includes_json_but_not_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -190,6 +202,9 @@ def test_make_rlist_command_includes_json_but_not_debug(): def test_make_rlist_command_includes_json(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -207,6 +222,9 @@ def test_make_rlist_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -222,6 +240,9 @@ def test_make_rlist_command_includes_lock_wait(): def test_make_rlist_command_includes_local_path(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -240,6 +261,9 @@ def test_make_rlist_command_includes_remote_path(): flexmock(module.flags).should_receive('make_flags').and_return( ('--remote-path', 'borg2') ).and_return(()).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -258,6 +282,9 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( ('--match-archives', 'sh:foo*') ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -271,8 +298,47 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') +def test_make_rlist_command_prefers_prefix_over_archive_name_format(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( + ('--match-archives', 'sh:foo*') + ) + flexmock(module.flags).should_receive('make_match_archives_flags').never() + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + ) + + assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') + + +def test_make_rlist_command_transforms_archive_name_format_into_match_archives(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + ) + + assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') + + def test_make_rlist_command_includes_short(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--short',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -301,6 +367,9 @@ def test_make_rlist_command_includes_short(): ) def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (f"--{argument_name.replace('_', '-')}", 'value') ) diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index a4814420..3628a1dc 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -12,6 +12,7 @@ def test_transfer_archives_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -41,6 +42,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): flexmock(module.flags).should_receive('make_flags').with_args('dry-run', True).and_return( ('--dry-run',) ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -67,6 +69,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -93,6 +96,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -123,6 +127,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'archive' ).and_return(('--match-archives', 'archive')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -137,7 +142,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive='archive', progress=None, match_archives=None, source_repository=None @@ -152,6 +157,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -166,7 +172,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None @@ -174,10 +180,40 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl ) +def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archives_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--match-archives', 'sh:bar-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, + borg_local_path='borg', + extra_environment=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + ) + + def test_transfer_archives_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -208,6 +244,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.flags).should_receive('make_flags').with_args( 'remote-path', 'borg2' ).and_return(('--remote-path', 'borg2')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -238,6 +275,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.flags).should_receive('make_flags').with_args('lock-wait', 5).and_return( ('--lock-wait', '5') ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) storage_config = {'lock_wait': 5} @@ -265,6 +303,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -293,6 +332,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') ) @@ -327,6 +367,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla flexmock(module.flags).should_receive('make_flags').with_args('other-repo', 'other').and_return( ('--other-repo', 'other') ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') From f256908b2709b643b4f7b27fd3fda150d3d32bca Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 31 Mar 2023 15:36:59 -0700 Subject: [PATCH 54/94] Document wording tweaks (#479). --- docs/how-to/make-per-application-backups.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 6e5d999a..fffd2c9d 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -91,7 +91,7 @@ in each configuration file. Under the hood, borgmatic accomplishes this by substituting globs for certain ephemeral data placeholders in your `archive_name_format`—and using the result -to filter archives for supported actions. +to filter archives when running supported actions. For instance, let's say that you have this in your configuration: @@ -101,7 +101,7 @@ location: archive_name_format: {hostname}-user-data-{now} ``` -borgmatic considers `{now}` an emphemeral placeholder that will probably +borgmatic considers `{now}` an emphemeral data placeholder that will probably change per archive, while `{hostname}` won't. So it turns the example value into `{hostname}-user-data-*` and applies it to filter down the set of archives used for actions like `rlist`, `info`, `prune`, `check`, etc. @@ -112,12 +112,11 @@ created for that application. Of course, this doesn't apply to actions like `compact` that operate on an entire repository. Prior to 1.7.11 The way to -limit the archives used was a `prefix` option in the `retention` section for -matching against the start of archive names used for a `prune` action and -a separate `prefix` option in the `consistency` section for matching against -the start of archive names used for a `check` action. Both of these options -are deprecated in favor of the auto-matching behavior in newer versions of -borgmatic mentioned above. +limit the archives used for the `prune` action was a `prefix` option in the +`retention` section for matching against the start of archive names. And the +option for limiting the archives used for the `check` action was a separate +`prefix` in the `consistency` section. Both of these options are deprecated in +favor of the auto-matching behavior in newer versions of borgmatic. ## Configuration includes From fa8bc285c8e82d8fd89327f5877c95957a8b4c76 Mon Sep 17 00:00:00 2001 From: kxxt Date: Sat, 1 Apr 2023 13:45:32 +0800 Subject: [PATCH 55/94] Fix randomly failing test. --- tests/integration/test_execute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 09e9093a..9c62941b 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -147,7 +147,7 @@ def test_log_outputs_kills_other_processes_when_one_errors(): ['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) flexmock(module).should_receive('exit_code_indicates_error').with_args( - other_process, None, 'borg' + ['sleep', '2'], None, 'borg' ).and_return(False) flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return( process.stdout From 6054ced931d35173627019054d9bc801cfd2d1ed Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 1 Apr 2023 22:10:32 +0530 Subject: [PATCH 56/94] fix: run typos --- NEWS | 6 +++--- borgmatic/borg/prune.py | 2 +- borgmatic/commands/arguments.py | 2 +- borgmatic/config/convert.py | 2 +- borgmatic/config/schema.yaml | 4 ++-- borgmatic/logger.py | 2 +- docs/_includes/components/toc.css | 2 +- ...backup-to-a-removable-drive-or-an-intermittent-server.md | 4 ++-- docs/how-to/develop-on-borgmatic.md | 4 ++-- docs/how-to/inspect-your-backups.md | 2 +- docs/how-to/set-up-backups.md | 2 +- tests/integration/config/test_validate.py | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/NEWS b/NEWS index 9bc9ebf6..c0ae3d26 100644 --- a/NEWS +++ b/NEWS @@ -388,7 +388,7 @@ configuration schema descriptions. 1.5.6 - * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the + * #292: Allow before_backup and similar hooks to exit with a soft failure without altering the monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring services with a "start" status until after before_* hooks finish. Failures in before_* hooks still trigger a monitoring "fail" status. @@ -457,7 +457,7 @@ * For "list" and "info" actions, show repository names even at verbosity 0. 1.4.22 - * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON ouput. + * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON output. * After a backup of a database dump in directory format, properly remove the dump directory. * In "borgmatic --help", don't expand $HOME in listing of default "--config" paths. @@ -829,7 +829,7 @@ * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files, editor swap files, etc. * #81: Document user-defined hooks run before/after backup, or on error. - * Add code style guidelines to the documention. + * Add code style guidelines to the documentation. 1.2.0 * #61: Support for Borg --list option via borgmatic command-line to list all archives. diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index d21ceee3..e9f665f8 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def make_prune_flags(retention_config, local_borg_version): ''' - Given a retention config dict mapping from option name to value, tranform it into an iterable of + Given a retention config dict mapping from option name to value, transform it into an iterable of command-line name-value flag pairs. For example, given a retention config of: diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 59751b08..44e11ebb 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -293,7 +293,7 @@ def make_parsers(): ) transfer_group.add_argument( '--upgrader', - help='Upgrader type used to convert the transfered data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion', + help='Upgrader type used to convert the transferred data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion', ) transfer_group.add_argument( '--progress', diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index e4e55e43..093ad0c1 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -43,7 +43,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): ] ) - # Split space-seperated values into actual lists, make "repository" into a list, and merge in + # Split space-separated values into actual lists, make "repository" into a list, and merge in # excludes. location = destination_config['location'] location['source_directories'] = source_config.location['source_directories'].split(' ') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3e94b8d2..fe673191 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -6,7 +6,7 @@ properties: constants: type: object description: | - Constants to use in the configuration file. All occurences of the + Constants to use in the configuration file. All occurrences of the constant name within culy braces will be replaced with the value. For example, if you have a constant named "hostname" with the value "myhostname", then the string "{hostname}" will be replaced with @@ -929,7 +929,7 @@ properties: type: string enum: ['sql'] description: | - Database dump output format. Currenly only "sql" + Database dump output format. Currently only "sql" is supported. Defaults to "sql" for a single database. Or, when database name is "all" and format is blank, dumps all databases to a single diff --git a/borgmatic/logger.py b/borgmatic/logger.py index e098bf96..648500b0 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -68,7 +68,7 @@ class Multi_stream_handler(logging.Handler): def emit(self, record): ''' - Dispatch the log record to the approriate stream handler for the record's log level. + Dispatch the log record to the appropriate stream handler for the record's log level. ''' self.log_level_to_handler[record.levelno].emit(record) diff --git a/docs/_includes/components/toc.css b/docs/_includes/components/toc.css index 039673f6..82cf15e5 100644 --- a/docs/_includes/components/toc.css +++ b/docs/_includes/components/toc.css @@ -94,7 +94,7 @@ display: block; } -/* Footer catgory navigation */ +/* Footer category navigation */ .elv-cat-list-active { font-weight: 600; } diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index 098b03f6..04ccbf79 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -116,8 +116,8 @@ There are some caveats you should be aware of with this feature. * You'll generally want to put a soft failure command in the `before_backup` hook, so as to gate whether the backup action occurs. While a soft failure is also supported in the `after_backup` hook, returning a soft failure there - won't prevent any actions from occuring, because they've already occurred! - Similiarly, you can return a soft failure from an `on_error` hook, but at + won't prevent any actions from occurring, because they've already occurred! + Similarly, you can return a soft failure from an `on_error` hook, but at that point it's too late to prevent the error. * Returning a soft failure does prevent further commands in the same hook from executing. So, like a standard error, it is an "early out". Unlike a standard diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index d4315457..b615ed00 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -101,10 +101,10 @@ the following deviations from it: * For strings, prefer single quotes over double quotes. * Limit all lines to a maximum of 100 characters. * Use trailing commas within multiline values or argument lists. - * For multiline constructs, put opening and closing delimeters on lines + * For multiline constructs, put opening and closing delimiters on lines separate from their contents. * Within multiline constructs, use standard four-space indentation. Don't align - indentation with an opening delimeter. + indentation with an opening delimiter. borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 57a2381c..67d8cc23 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -111,7 +111,7 @@ By default, borgmatic logs to a local syslog-compatible daemon if one is present and borgmatic is running in a non-interactive console. Where those logs show up depends on your particular system. If you're using systemd, try running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or -similiar. +similar. You can customize the log level used for syslog logging with the `--syslog-verbosity` flag, and this is independent from the console logging diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 52962c34..fb3eb91f 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -157,7 +157,7 @@ variable or set the `BORG_PASSPHRASE` environment variable. See the section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption) of the Borg Quick Start for more info. -Alternatively, you can specify the passphrase programatically by setting +Alternatively, you can specify the passphrase programmatically by setting either the borgmatic `encryption_passcommand` configuration variable or the `BORG_PASSCOMMAND` environment variable. See the [Borg Security FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically) diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 67e510ab..87428ddc 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -8,7 +8,7 @@ from flexmock import flexmock from borgmatic.config import validate as module -def test_schema_filename_returns_plausable_path(): +def test_schema_filename_returns_plausible_path(): schema_path = module.schema_filename() assert schema_path.endswith('/schema.yaml') From 2934d0902c8ec26b4d1cc2c93e4a46f7801eeb67 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Apr 2023 11:03:59 -0700 Subject: [PATCH 57/94] Code spell checking on every test run! --- borgmatic/borg/check.py | 2 +- borgmatic/config/schema.yaml | 16 ++++++++-------- docs/how-to/develop-on-borgmatic.md | 8 ++++++++ setup.cfg | 3 +++ test_requirements.txt | 1 + tox.ini | 6 ++++++ 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 4630782a..2c7c8fc5 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -147,7 +147,7 @@ def filter_checks_on_frequency( def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None): ''' - Given the local Borg version, a storge configuration dict, a parsed sequence of checks, the + Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the check last value, and a consistency check prefix, transform the checks into tuple of command-line flags. diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 875c05c3..04b3c6ec 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -930,14 +930,14 @@ properties: type: string enum: ['sql'] description: | - Database dump output format. Currently only "sql" - is supported. Defaults to "sql" for a single - database. Or, when database name is "all" and - format is blank, dumps all databases to a single - file. But if a format is specified with an "all" - database name, dumps each database to a separate - file of that format, allowing more convenient - restores of individual databases. + Database dump output format. Currently only + "sql" is supported. Defaults to "sql" for a + single database. Or, when database name is "all" + and format is blank, dumps all databases to a + single file. But if a format is specified with + an "all" database name, dumps each database to a + separate file of that format, allowing more + convenient restores of individual databases. example: directory add_drop_database: type: boolean diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 33346fae..ebc79c6f 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -73,6 +73,14 @@ can ask isort to order your imports for you: tox -e isort ``` +Similarly, if you get errors about spelling mistakes in source code, you can +ask codespell to correct them: + +```bash +tox -e codespell +``` + + ### End-to-end tests borgmatic additionally includes some end-to-end tests that integration test diff --git a/setup.cfg b/setup.cfg index 67b989dc..f97ae972 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,3 +20,6 @@ known_first_party = borgmatic line_length = 100 multi_line_output = 3 skip = .tox + +[codespell] +skip = .git,.tox,build diff --git a/test_requirements.txt b/test_requirements.txt index d34bc623..7910788b 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -2,6 +2,7 @@ appdirs==1.4.4; python_version >= '3.8' attrs==20.3.0; python_version >= '3.8' black==19.10b0; python_version >= '3.8' click==7.1.2; python_version >= '3.8' +codespell==2.2.4 colorama==0.4.4 coverage==5.3 flake8==4.0.1 diff --git a/tox.ini b/tox.ini index 17b7a9da..b81588af 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,7 @@ commands = py38,py39,py310,py311: black --check . isort --check-only --settings-path setup.cfg . flake8 borgmatic tests + codespell [testenv:black] commands = @@ -35,3 +36,8 @@ commands = deps = {[testenv]deps} commands = isort --settings-path setup.cfg . + +[testenv:codespell] +deps = {[testenv]deps} +commands = + codespell --write-changes From b9328e6d4273a3ad7a1ef4d990888584964f0a0c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Apr 2023 14:09:48 -0700 Subject: [PATCH 58/94] Add spellchecking of source code to NEWS. --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index cce27406..af0fe9ac 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,7 @@ auto-matching behavior (see above). * #662: Fix regression in which "check_repositories" option failed to match repositories. * #663: Fix regression in which the "transfer" action produced a traceback. + * Add spellchecking of source code during test runs. 1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. From 275e99d0b9dcbe0ab3509bdbbd10df1dad37c10c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Apr 2023 14:38:52 -0700 Subject: [PATCH 59/94] Add codespell link to documentation. --- docs/how-to/develop-on-borgmatic.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index ebc79c6f..6699bdf3 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -74,7 +74,8 @@ tox -e isort ``` Similarly, if you get errors about spelling mistakes in source code, you can -ask codespell to correct them: +ask [codespell](https://github.com/codespell-project/codespell) to correct +them: ```bash tox -e codespell From 9712d00680fd0f8ec77e506d561745161e9cac04 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Apr 2023 23:57:55 -0700 Subject: [PATCH 60/94] Add "match_archives" option (#588). --- NEWS | 11 ++++--- borgmatic/borg/check.py | 4 ++- borgmatic/borg/flags.py | 21 ++++++++----- borgmatic/borg/info.py | 4 ++- borgmatic/borg/prune.py | 4 ++- borgmatic/borg/rlist.py | 4 ++- borgmatic/borg/transfer.py | 4 ++- borgmatic/config/schema.yaml | 11 +++++++ docs/how-to/make-per-application-backups.md | 20 +++++++++++- tests/unit/borg/test_check.py | 2 +- tests/unit/borg/test_flags.py | 35 +++++++++++++++------ tests/unit/borg/test_info.py | 28 ++++++++--------- tests/unit/borg/test_prune.py | 2 +- tests/unit/borg/test_rlist.py | 24 +++++++------- tests/unit/borg/test_transfer.py | 2 +- 15 files changed, 120 insertions(+), 56 deletions(-) diff --git a/NEWS b/NEWS index af0fe9ac..739ce86c 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,11 @@ 1.7.11.dev0 - * #479: Automatically use the "archive_name_format" option to filter which archives get used for - borgmatic actions that operate on multiple archives. See the documentation for more information: + * #479, #588: Automatically use the "archive_name_format" option to filter which archives get used + for borgmatic actions that operate on multiple archives. Override this behavior with the new + "match_archives" option in the storage section. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming - * #479: The "prefix" options have been deprecated in favor of the new "archive_name_format" - auto-matching behavior (see above). - * #662: Fix regression in which "check_repositories" option failed to match repositories. + * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format" + auto-matching behavior and the "match_archives" option. + * #662: Fix regression in which the "check_repositories" option failed to match repositories. * #663: Fix regression in which the "transfer" action produced a traceback. * Add spellchecking of source code during test runs. diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 2c7c8fc5..cee9d923 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -183,7 +183,9 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None if prefix else ( flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) ) ) diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 1354d658..986531bc 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -59,18 +59,25 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version): ) -def make_match_archives_flags(archive_name_format, local_borg_version): +def make_match_archives_flags(match_archives, archive_name_format, local_borg_version): ''' - Return the match archives flags that would match archives created with the given archive name - format (if any). This is done by replacing certain archive name format placeholders for - ephemeral data (like "{now}") with globs. + Return match archives flags based on the given match archives value, if any. If it isn't set, + return match archives flags to match archives created with the given archive name format, if + any. This is done by replacing certain archive name format placeholders for ephemeral data (like + "{now}") with globs. ''' + if match_archives: + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): + return ('--match-archives', match_archives) + else: + return ('--glob-archives', re.sub(r'^sh:', '', match_archives)) + if not archive_name_format: return () - match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format) + derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format) if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): - return ('--match-archives', f'sh:{match_archives}') + return ('--match-archives', f'sh:{derived_match_archives}') else: - return ('--glob-archives', f'{match_archives}') + return ('--glob-archives', f'{derived_match_archives}') diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index fa59165c..a7ed98ed 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -46,7 +46,9 @@ def display_archives_info( if info_arguments.prefix else ( flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) ) ) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 11ba308e..7e12b9c2 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -39,7 +39,9 @@ def make_prune_flags(storage_config, retention_config, local_borg_version): return tuple( element for pair in flag_pairs for element in pair ) + flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index f3935a16..f84499ce 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -96,7 +96,9 @@ def make_rlist_command( if rlist_arguments.prefix else ( flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) ) ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index a350eff7..78a76124 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -41,7 +41,9 @@ def transfer_archives( ) or ( flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) ) ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 04b3c6ec..26500855 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -382,6 +382,17 @@ properties: actions like rlist, info, or check, borgmatic automatically tries to match only archives created with this name format. example: "{hostname}-documents-{now}" + match_archives: + type: string + description: | + A Borg pattern for filtering down the archives used by + borgmatic actions that operate on multiple archives. For + Borg 1.x, use a shell pattern here and see the output of + "borg help placeholders" for details. For Borg 2.x, see the + output of "borg help match-archives". If match_archives is + not specified, borgmatic defaults to deriving the + match_archives value from archive_name_format. + example: "sh:{hostname}-*" relocated_repo_access_is_ok: type: boolean description: | diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index fffd2c9d..a4898430 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -111,12 +111,30 @@ application-specific configuration file, it only operates on the archives created for that application. Of course, this doesn't apply to actions like `compact` that operate on an entire repository. +If this behavior isn't quite smart enough for your needs, you can use the +`match_archives` option to override the pattern that borgmatic uses for +filtering archives. For example: + +```yaml +location: + ... + archive_name_format: {hostname}-user-data-{now} + match_archives: sh:myhost-user-data-* +``` + +For Borg 1.x, use a shell pattern for the `match_archives` value and see the +[Borg patterns +documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns) +for more information. For Borg 2.x, see the [match archives +documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/help.html#borg-help-match-archives). + Prior to 1.7.11 The way to limit the archives used for the `prune` action was a `prefix` option in the `retention` section for matching against the start of archive names. And the option for limiting the archives used for the `check` action was a separate `prefix` in the `consistency` section. Both of these options are deprecated in -favor of the auto-matching behavior in newer versions of borgmatic. +favor of the auto-matching behavior (or `match_archives`) in newer versions of +borgmatic. ## Configuration includes diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 7c233bcf..65e17291 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -320,7 +320,7 @@ def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_fla def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '1.2.3' # noqa: FS003 + None, 'bar-{now}', '1.2.3' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) flags = module.make_check_flags( diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index cf9eedbb..86247040 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -82,32 +82,49 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a @pytest.mark.parametrize( - 'archive_name_format,feature_available,expected_result', + 'match_archives, archive_name_format,feature_available,expected_result', ( - (None, True, ()), - ('', True, ()), + (None, None, True, ()), + (None, '', True, ()), + ('re:foo-.*', '{hostname}-{now}', True, ('--match-archives', 're:foo-.*'),), # noqa: FS003 + ('sh:foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003 + ('foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003 ( + None, '{hostname}-docs-{now}', # noqa: FS003 True, ('--match-archives', 'sh:{hostname}-docs-*'), # noqa: FS003 ), - ('{utcnow}-docs-{user}', True, ('--match-archives', 'sh:*-docs-{user}')), # noqa: FS003 - ('{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003 ( + None, + '{utcnow}-docs-{user}', # noqa: FS003 + True, + ('--match-archives', 'sh:*-docs-{user}'), # noqa: FS003 + ), + (None, '{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003 + ( + None, 'stuff-{now:%Y-%m-%dT%H:%M:%S.%f}', # noqa: FS003 True, ('--match-archives', 'sh:stuff-*'), ), - ('{hostname}-docs-{now}', False, ('--glob-archives', '{hostname}-docs-*')), # noqa: FS003 - ('{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003 + ( + None, + '{hostname}-docs-{now}', # noqa: FS003 + False, + ('--glob-archives', '{hostname}-docs-*'), # noqa: FS003 + ), + (None, '{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003 ), ) def test_make_match_archives_flags_makes_flags_with_globs( - archive_name_format, feature_available, expected_result + match_archives, archive_name_format, feature_available, expected_result ): flexmock(module.feature).should_receive('available').and_return(feature_available) assert ( - module.make_match_archives_flags(archive_name_format, local_borg_version=flexmock()) + module.make_match_archives_flags( + match_archives, archive_name_format, local_borg_version=flexmock() + ) == expected_result ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index b25f29f6..ee525857 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -13,7 +13,7 @@ def test_display_archives_info_calls_borg_with_parameters(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -38,7 +38,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -63,7 +63,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -88,7 +88,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -114,7 +114,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -139,7 +139,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -166,7 +166,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param 'match-archives', 'archive' ).and_return(('--match-archives', 'archive')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -191,7 +191,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -220,7 +220,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para 'remote-path', 'borg1' ).and_return(('--remote-path', 'borg1')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -249,7 +249,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete ('--lock-wait', '5') ) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -278,7 +278,7 @@ def test_display_archives_info_transforms_prefix_into_match_archives_parameters( 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -306,7 +306,7 @@ def test_display_archives_info_prefers_prefix_over_archive_name_format(): 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -331,7 +331,7 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '2.3.4' # noqa: FS003 + None, 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -358,7 +358,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 1c8843e2..37731aca 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -74,7 +74,7 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): retention_config = OrderedDict((('keep_daily', 1), ('prefix', None))) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '1.2.3' # noqa: FS003 + None, 'bar-{now}', '1.2.3' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 8a10b075..136821d6 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -128,7 +128,7 @@ def test_make_rlist_command_includes_log_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -147,7 +147,7 @@ def test_make_rlist_command_includes_json_but_not_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -166,7 +166,7 @@ def test_make_rlist_command_includes_log_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -185,7 +185,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -203,7 +203,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): def test_make_rlist_command_includes_json(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -223,7 +223,7 @@ def test_make_rlist_command_includes_lock_wait(): ('--lock-wait', '5') ).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -241,7 +241,7 @@ def test_make_rlist_command_includes_lock_wait(): def test_make_rlist_command_includes_local_path(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -262,7 +262,7 @@ def test_make_rlist_command_includes_remote_path(): ('--remote-path', 'borg2') ).and_return(()).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -283,7 +283,7 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): ('--match-archives', 'sh:foo*') ) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -319,7 +319,7 @@ def test_make_rlist_command_prefers_prefix_over_archive_name_format(): def test_make_rlist_command_transforms_archive_name_format_into_match_archives(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '1.2.3' # noqa: FS003 + None, 'bar-{now}', '1.2.3' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -337,7 +337,7 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives() def test_make_rlist_command_includes_short(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--short',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -368,7 +368,7 @@ def test_make_rlist_command_includes_short(): def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (f"--{argument_name.replace('_', '-')}", 'value') diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 3628a1dc..857c86ec 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -185,7 +185,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '2.3.4' # noqa: FS003 + None, 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) From 01811e03ba1c46cbf04acbbe3bde99b3433e57f9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 2 Apr 2023 14:38:35 -0700 Subject: [PATCH 61/94] Tagged the auto-matching archive behavior as breaking in NEWS. --- NEWS | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 739ce86c..908256a3 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,9 @@ 1.7.11.dev0 - * #479, #588: Automatically use the "archive_name_format" option to filter which archives get used - for borgmatic actions that operate on multiple archives. Override this behavior with the new - "match_archives" option in the storage section. See the documentation for more information: + * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives + get used for borgmatic actions that operate on multiple archives. Override this behavior with the + new "match_archives" option in the storage section. This change is "breaking" in that it silently + changes which archives get considered for "rlist", "prune", "check", etc. See the documentation + for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format" auto-matching behavior and the "match_archives" option. From 7e6bee84b0aa99aa0dece40fd6ee416d1c038674 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 2 Apr 2023 23:06:36 -0700 Subject: [PATCH 62/94] Add "--log-file-format" flag for customizing the log message format (#658). --- NEWS | 3 +++ borgmatic/commands/arguments.py | 8 +++--- borgmatic/commands/borgmatic.py | 1 + borgmatic/logger.py | 11 +++++++-- docs/how-to/inspect-your-backups.md | 38 +++++++++++++++++++++++++++-- tests/unit/test_logger.py | 33 +++++++++++++++++++++++-- 6 files changed, 85 insertions(+), 9 deletions(-) diff --git a/NEWS b/NEWS index 908256a3..dd0a6b65 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,9 @@ https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format" auto-matching behavior and the "match_archives" option. + * #658: Add "--log-file-format" flag for customizing the log message format. See the documentation + for more information: + https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#logging-to-file * #662: Fix regression in which the "check_repositories" option failed to match repositories. * #663: Fix regression in which the "transfer" action produced a traceback. * Add spellchecking of source code during test runs. diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 44e11ebb..84ab79fc 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -178,10 +178,12 @@ def make_parsers(): help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)', ) global_group.add_argument( - '--log-file', + '--log-file', type=str, help='Write log messages to this file instead of syslog', + ) + global_group.add_argument( + '--log-file-format', type=str, - default=None, - help='Write log messages to this file instead of syslog', + help='Log format string used for log messages written to the log file', ) global_group.add_argument( '--override', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b4aadfca..22aba0b1 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -700,6 +700,7 @@ def main(): # pragma: no cover verbosity_to_log_level(global_arguments.log_file_verbosity), verbosity_to_log_level(global_arguments.monitoring_verbosity), global_arguments.log_file, + global_arguments.log_file_format, ) except (FileNotFoundError, PermissionError) as error: configure_logging(logging.CRITICAL) diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 648500b0..52065928 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -156,6 +156,7 @@ def configure_logging( log_file_log_level=None, monitoring_log_level=None, log_file=None, + log_file_format=None, ): ''' Configure logging to go to both the console and (syslog or log file). Use the given log levels, @@ -200,12 +201,18 @@ def configure_logging( if syslog_path and not interactive_console(): syslog_handler = logging.handlers.SysLogHandler(address=syslog_path) - syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s')) + syslog_handler.setFormatter( + logging.Formatter('borgmatic: {levelname} {message}', style='{') # noqa: FS003 + ) syslog_handler.setLevel(syslog_log_level) handlers = (console_handler, syslog_handler) elif log_file: file_handler = logging.handlers.WatchedFileHandler(log_file) - file_handler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s')) + file_handler.setFormatter( + logging.Formatter( + log_file_format or '[{asctime}] {levelname}: {message}', style='{' # noqa: FS003 + ) + ) file_handler.setLevel(log_file_log_level) handlers = (console_handler, file_handler) else: diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 67d8cc23..b6238fe9 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -154,5 +154,39 @@ borgmatic --log-file /path/to/file.log Note that if you use the `--log-file` flag, you are responsible for rotating the log file so it doesn't grow too large, for example with -[logrotate](https://wiki.archlinux.org/index.php/Logrotate). Also, there is a -`--log-file-verbosity` flag to customize the log file's log level. +[logrotate](https://wiki.archlinux.org/index.php/Logrotate). + +You can the `--log-file-verbosity` flag to customize the log file's log level: + +```bash +borgmatic --log-file /path/to/file.log --log-file-verbosity 2 +``` + +New in borgmatic version 1.7.11 +Use the `--log-file-format` flag to override the default log message format. +This format string can contain a series of named placeholders wrapped in curly +brackets. For instance, the default log format is: `[{asctime}] {levelname}: +{message}`. This means each log message is recorded as the log time (in square +brackets), a logging level name, a colon, and the actual log message. + +So if you just want each log message to get logged *without* a timestamp or a +logging level name: + +```bash +borgmatic --log-file /path/to/file.log --log-file-format "{message}" +``` + +Here is a list of available placeholders: + + * `{asctime}`: time the log message was created + * `{levelname}`: level of the log message (`INFO`, `DEBUG`, etc.) + * `{lineno}`: line number in the source file where the log message originated + * `{message}`: actual log message + * `{pathname}`: path of the source file where the log message originated + +See the [Python logging +documentation](https://docs.python.org/3/library/logging.html#logrecord-attributes) +for additional placeholders. + +Note that this `--log-file-format` flg only applies to the specified +`--log-file` and not to syslog or other logging. diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 0fb284a3..dc9c748d 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -285,7 +285,7 @@ def test_configure_logging_skips_syslog_if_interactive_console(): module.configure_logging(console_log_level=logging.INFO) -def test_configure_logging_to_logfile_instead_of_syslog(): +def test_configure_logging_to_log_file_instead_of_syslog(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER flexmock(module).should_receive('Multi_stream_handler').and_return( @@ -309,7 +309,36 @@ def test_configure_logging_to_logfile_instead_of_syslog(): ) -def test_configure_logging_skips_logfile_if_argument_is_none(): +def test_configure_logging_to_log_file_formats_with_custom_log_format(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.ANSWER + flexmock(module.logging).should_receive('Formatter').with_args( + '{message}', style='{' # noqa: FS003 + ).once() + flexmock(module).should_receive('Multi_stream_handler').and_return( + flexmock(setFormatter=lambda formatter: None, setLevel=lambda level: None) + ) + + flexmock(module).should_receive('interactive_console').and_return(False) + flexmock(module.logging).should_receive('basicConfig').with_args( + level=logging.DEBUG, handlers=tuple + ) + flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True) + flexmock(module.logging.handlers).should_receive('SysLogHandler').never() + file_handler = logging.handlers.WatchedFileHandler('/tmp/logfile') + flexmock(module.logging.handlers).should_receive('WatchedFileHandler').with_args( + '/tmp/logfile' + ).and_return(file_handler).once() + + module.configure_logging( + console_log_level=logging.INFO, + log_file_log_level=logging.DEBUG, + log_file='/tmp/logfile', + log_file_format='{message}', # noqa: FS003 + ) + + +def test_configure_logging_skips_log_file_if_argument_is_none(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER flexmock(module).should_receive('Multi_stream_handler').and_return( From fc58ba5763d37c9b8f1c2d9e9fe6a275911eb2e5 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 3 Apr 2023 17:36:24 +0530 Subject: [PATCH 63/94] add favicon to documentation --- docs/_includes/layouts/base.njk | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_includes/layouts/base.njk b/docs/_includes/layouts/base.njk index be588e9b..dbb19d99 100644 --- a/docs/_includes/layouts/base.njk +++ b/docs/_includes/layouts/base.njk @@ -3,6 +3,7 @@ + {{ subtitle + ' - ' if subtitle}}{{ title }} {%- set css %} {% include 'index.css' %} From 00d1dea94ea826d8d91d5f6d936c70638889bb32 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 3 Apr 2023 16:11:25 -0700 Subject: [PATCH 64/94] Bump version for release. --- NEWS | 2 +- docs/how-to/inspect-your-backups.md | 6 +++--- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index dd0a6b65..f693e25f 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.11.dev0 +1.7.11 * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives get used for borgmatic actions that operate on multiple archives. Override this behavior with the new "match_archives" option in the storage section. This change is "breaking" in that it silently diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index b6238fe9..73020eda 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -162,9 +162,9 @@ You can the `--log-file-verbosity` flag to customize the log file's log level: borgmatic --log-file /path/to/file.log --log-file-verbosity 2 ``` -New in borgmatic version 1.7.11 -Use the `--log-file-format` flag to override the default log message format. -This format string can contain a series of named placeholders wrapped in curly +New in version 1.7.11 Use the +`--log-file-format` flag to override the default log message format. This +format string can contain a series of named placeholders wrapped in curly brackets. For instance, the default log format is: `[{asctime}] {levelname}: {message}`. This means each log message is recorded as the log time (in square brackets), a logging level name, a colon, and the actual log message. diff --git a/setup.py b/setup.py index bc4e4b75..ad58f0e5 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.11.dev0' +VERSION = '1.7.11' setup( From 616eb6b6daa5827395da1533ad4423fb2ce450aa Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 4 Apr 2023 21:25:10 -0700 Subject: [PATCH 65/94] Fix error with "info --match-archives" and fix "--match-archives" overriding logic (#666). --- NEWS | 5 + borgmatic/borg/info.py | 11 +- borgmatic/borg/rlist.py | 4 +- borgmatic/borg/transfer.py | 10 +- borgmatic/commands/arguments.py | 20 +++- docs/how-to/make-per-application-backups.md | 8 +- setup.py | 2 +- tests/integration/borg/test_commands.py | 104 +++++++++++++++++++ tests/integration/commands/test_arguments.py | 14 +++ tests/unit/borg/test_info.py | 90 ++++++++++++---- tests/unit/borg/test_rlist.py | 81 +++++++++++---- tests/unit/borg/test_transfer.py | 12 +-- 12 files changed, 292 insertions(+), 69 deletions(-) create mode 100644 tests/integration/borg/test_commands.py diff --git a/NEWS b/NEWS index f693e25f..96634fd0 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.7.12.dev0 + * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the + "--match-archives" flag to correctly override the "match_archives" configuration option for + the "transfer", "list", "rlist", and "info" actions. + 1.7.11 * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives get used for borgmatic actions that operate on multiple archives. Override this behavior with the diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index a7ed98ed..80739c07 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -46,21 +46,18 @@ def display_archives_info( if info_arguments.prefix else ( flags.make_match_archives_flags( - storage_config.get('match_archives'), + info_arguments.match_archives + or info_arguments.archive + or storage_config.get('match_archives'), storage_config.get('archive_name_format'), local_borg_version, ) ) ) + flags.make_flags_from_arguments( - info_arguments, excludes=('repository', 'archive', 'prefix') + info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives') ) + flags.make_repository_flags(repository_path, local_borg_version) - + ( - flags.make_flags('match-archives', info_arguments.archive) - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) - else flags.make_flags('glob-archives', info_arguments.archive) - ) ) if info_arguments.json: diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index f84499ce..9686b67b 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -52,7 +52,7 @@ def resolve_archive_name( return latest_archive -MAKE_FLAGS_EXCLUDES = ('repository', 'prefix') +MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives') def make_rlist_command( @@ -96,7 +96,7 @@ def make_rlist_command( if rlist_arguments.prefix else ( flags.make_match_archives_flags( - storage_config.get('match_archives'), + rlist_arguments.match_archives or storage_config.get('match_archives'), storage_config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 78a76124..9fd05b76 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -28,12 +28,6 @@ def transfer_archives( + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + flags.make_flags('lock-wait', storage_config.get('lock_wait', None)) - + (('--progress',) if transfer_arguments.progress else ()) - + ( - flags.make_flags( - 'match-archives', transfer_arguments.match_archives or transfer_arguments.archive - ) - ) + ( flags.make_flags_from_arguments( transfer_arguments, @@ -41,7 +35,9 @@ def transfer_archives( ) or ( flags.make_match_archives_flags( - storage_config.get('match_archives'), + transfer_arguments.match_archives + or transfer_arguments.archive + or storage_config.get('match_archives'), storage_config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 84ab79fc..13ee3153 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -652,7 +652,7 @@ def make_parsers(): '--json', default=False, action='store_true', help='Output results as JSON' ) rlist_group.add_argument( - '-P', '--prefix', help='Only list archive names starting with this prefix' + '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix' ) rlist_group.add_argument( '-a', @@ -707,7 +707,7 @@ def make_parsers(): '--json', default=False, action='store_true', help='Output results as JSON' ) list_group.add_argument( - '-P', '--prefix', help='Only list archive names starting with this prefix' + '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix' ) list_group.add_argument( '-a', @@ -779,7 +779,9 @@ def make_parsers(): '--json', dest='json', default=False, action='store_true', help='Output results as JSON' ) info_group.add_argument( - '-P', '--prefix', help='Only show info for archive names starting with this prefix' + '-P', + '--prefix', + help='Deprecated. Only show info for archive names starting with this prefix', ) info_group.add_argument( '-a', @@ -877,7 +879,17 @@ def parse_arguments(*unparsed_arguments): and arguments['transfer'].match_archives ): raise ValueError( - 'With the transfer action, only one of --archive and --glob-archives flags can be used.' + 'With the transfer action, only one of --archive and --match-archives flags can be used.' + ) + + if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives): + raise ValueError( + 'With the list action, only one of --prefix or --match-archives flags can be used.' + ) + + if 'rlist' in arguments and (arguments['rlist'].prefix and arguments['rlist'].match_archives): + raise ValueError( + 'With the rlist action, only one of --prefix or --match-archives flags can be used.' ) if 'info' in arguments and ( diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index a4898430..27c643f9 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -128,13 +128,17 @@ documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg- for more information. For Borg 2.x, see the [match archives documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/help.html#borg-help-match-archives). +Some borgmatic command-line actions also have a `--match-archives` flag that +overrides both the auto-matching behavior and the `match_archives` +configuration option. + Prior to 1.7.11 The way to limit the archives used for the `prune` action was a `prefix` option in the `retention` section for matching against the start of archive names. And the option for limiting the archives used for the `check` action was a separate `prefix` in the `consistency` section. Both of these options are deprecated in -favor of the auto-matching behavior (or `match_archives`) in newer versions of -borgmatic. +favor of the auto-matching behavior (or `match_archives`/`--match-archives`) +in newer versions of borgmatic. ## Configuration includes diff --git a/setup.py b/setup.py index ad58f0e5..4b870eb4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.11' +VERSION = '1.7.12.dev0' setup( diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py new file mode 100644 index 00000000..4f9a6b25 --- /dev/null +++ b/tests/integration/borg/test_commands.py @@ -0,0 +1,104 @@ +import copy + +import flexmock + +import borgmatic.borg.info +import borgmatic.borg.list +import borgmatic.borg.rlist +import borgmatic.borg.transfer +import borgmatic.commands.arguments + + +def assert_command_does_not_duplicate_flags(command, *args, **kwargs): + ''' + Assert that the given Borg command sequence does not contain any duplicated flags, e.g. + "--match-archives" twice anywhere in the command. + ''' + flag_counts = {} + + for flag_name in command: + if not flag_name.startswith('--'): + continue + + if flag_name in flag_counts: + flag_counts[flag_name] += 1 + else: + flag_counts[flag_name] = 1 + + assert flag_counts == { + flag_name: 1 for flag_name in flag_counts + }, f"Duplicate flags found in: {' '.join(command)}" + + +def fuzz_argument(arguments, argument_name): + ''' + Given an argparse.Namespace instance of arguments and an argument name in it, copy the arguments + namespace and set the argument name in the copy with a fake value. Return the copied arguments. + + This is useful for "fuzzing" a unit under test by passing it each possible argument in turn, + making sure it doesn't blow up or duplicate Borg arguments. + ''' + arguments_copy = copy.copy(arguments) + value = getattr(arguments_copy, argument_name) + setattr(arguments_copy, argument_name, not value if isinstance(value, bool) else 'value') + + return arguments_copy + + +def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments( + 'transfer', '--source-repository', 'foo' + )['transfer'] + flexmock(borgmatic.borg.transfer).should_receive('execute_command').replace_with( + assert_command_does_not_duplicate_flags + ) + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.transfer.transfer_archives( + False, 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) + + +def test_make_list_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('list')['list'] + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.list.make_list_command( + 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) + + +def test_make_rlist_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('rlist')['rlist'] + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.rlist.make_rlist_command( + 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) + + +def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('info')['info'] + flexmock(borgmatic.borg.info).should_receive('execute_command_and_capture_output').replace_with( + assert_command_does_not_duplicate_flags + ) + flexmock(borgmatic.borg.info).should_receive('execute_command').replace_with( + assert_command_does_not_duplicate_flags + ) + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.info.display_archives_info( + 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 754564b2..1fc8f8c8 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -465,6 +465,20 @@ def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives ) +def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + + +def test_parse_arguments_disallows_rlist_with_both_prefix_and_match_archives(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('rlist', '--prefix', 'foo', '--match-archives', 'sh:*bar') + + def test_parse_arguments_disallows_info_with_both_archive_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index ee525857..d51bf14d 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -29,7 +29,7 @@ def test_display_archives_info_calls_borg_with_parameters(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -54,7 +54,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -77,7 +77,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=True, prefix=None), + info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) assert json_output == '[]' @@ -105,7 +105,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -128,7 +128,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=True, prefix=None), + info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) assert json_output == '[]' @@ -152,7 +152,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=True, prefix=None), + info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) assert json_output == '[]' @@ -162,17 +162,14 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags').with_args( - 'match-archives', 'archive' - ).and_return(('--match-archives', 'archive')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, None, '2.3.4' - ).and_return(()) + 'archive', None, '2.3.4' + ).and_return(('--match-archives', 'archive')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'info', '--repo', 'repo', '--match-archives', 'archive'), + ('borg', 'info', '--match-archives', 'archive', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, borg_local_path='borg', extra_environment=None, @@ -182,7 +179,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive='archive', json=False, prefix=None), + info_arguments=flexmock(archive='archive', json=False, prefix=None, match_archives=None), ) @@ -207,7 +204,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), local_path='borg1', ) @@ -236,7 +233,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), remote_path='borg1', ) @@ -266,7 +263,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -347,11 +344,64 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) -@pytest.mark.parametrize('argument_name', ('match_archives', 'sort_by', 'first', 'last')) +def test_display_archives_with_match_archives_option_calls_borg_with_match_archives_parameter(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'sh:foo-*', 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:foo-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={ + 'archive_name_format': 'bar-{now}', # noqa: FS003 + 'match_archives': 'sh:foo-*', + }, + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), + ) + + +def test_display_archives_with_match_archives_flag_calls_borg_with_match_archives_parameter(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'sh:foo-*', 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:foo-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), + ) + + +@pytest.mark.parametrize('argument_name', ('sort_by', 'first', 'last')) def test_display_archives_info_passes_through_arguments_to_borg(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -376,5 +426,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None, **{argument_name: 'value'}), + info_arguments=flexmock( + archive=None, json=False, prefix=None, match_archives=None, **{argument_name: 'value'} + ), ) diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 136821d6..6b1561a2 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -137,7 +137,9 @@ def test_make_rlist_command_includes_log_info(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--info', 'repo') @@ -156,7 +158,9 @@ def test_make_rlist_command_includes_json_but_not_info(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=True, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--json', 'repo') @@ -175,7 +179,9 @@ def test_make_rlist_command_includes_log_debug(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') @@ -194,7 +200,9 @@ def test_make_rlist_command_includes_json_but_not_debug(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=True, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--json', 'repo') @@ -212,7 +220,9 @@ def test_make_rlist_command_includes_json(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=True, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--json', 'repo') @@ -232,7 +242,9 @@ def test_make_rlist_command_includes_lock_wait(): repository_path='repo', storage_config={'lock_wait': 5}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') @@ -250,7 +262,9 @@ def test_make_rlist_command_includes_local_path(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), local_path='borg2', ) @@ -271,7 +285,9 @@ def test_make_rlist_command_includes_remote_path(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), remote_path='borg2', ) @@ -328,7 +344,9 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives() repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') @@ -346,7 +364,9 @@ def test_make_rlist_command_includes_short(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None, short=True), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None, short=True + ), ) assert command == ('borg', 'list', '--short', 'repo') @@ -354,16 +374,7 @@ def test_make_rlist_command_includes_short(): @pytest.mark.parametrize( 'argument_name', - ( - 'match_archives', - 'sort_by', - 'first', - 'last', - 'exclude', - 'exclude_from', - 'pattern', - 'patterns_from', - ), + ('sort_by', 'first', 'last', 'exclude', 'exclude_from', 'pattern', 'patterns_from',), ) def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) @@ -384,6 +395,7 @@ def test_make_rlist_command_includes_additional_flags(argument_name): paths=None, json=False, prefix=None, + match_archives=None, find_paths=None, format=None, **{argument_name: 'value'}, @@ -393,6 +405,35 @@ def test_make_rlist_command_includes_additional_flags(argument_name): assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') +def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_parameters(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'foo-*', None, '1.2.3', + ).and_return(('--match-archives', 'foo-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock( + archive=None, + paths=None, + json=False, + prefix=None, + match_archives='foo-*', + find_paths=None, + format=None, + ), + ) + + assert command == ('borg', 'list', '--match-archives', 'foo-*', 'repo') + + def test_list_repository_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 857c86ec..8f41bf5a 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -124,10 +124,9 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags').with_args( - 'match-archives', 'archive' + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'archive', 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'archive')) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -154,10 +153,9 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags').with_args( - 'match-archives', 'sh:foo*' + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'sh:foo*', 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'sh:foo*')) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -304,7 +302,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( From a9a65ebe54d4d915cf16ba44358de5aeb7e83818 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 4 Apr 2023 22:11:36 -0700 Subject: [PATCH 66/94] Fix integration tests to actually assert (#666). --- tests/integration/borg/test_commands.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 4f9a6b25..49cd0e96 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -69,10 +69,12 @@ def test_make_list_command_does_not_duplicate_flags_or_raise(): if argument_name.startswith('_'): continue - borgmatic.borg.list.make_list_command( + command = borgmatic.borg.list.make_list_command( 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) ) + assert_command_does_not_duplicate_flags(command) + def test_make_rlist_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments('rlist')['rlist'] @@ -81,10 +83,12 @@ def test_make_rlist_command_does_not_duplicate_flags_or_raise(): if argument_name.startswith('_'): continue - borgmatic.borg.rlist.make_rlist_command( + command = borgmatic.borg.rlist.make_rlist_command( 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) ) + assert_command_does_not_duplicate_flags(command) + def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments('info')['info'] From 9bc2322f9a38c6ab704be284563b1928c7673172 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 6 Apr 2023 02:10:36 +0530 Subject: [PATCH 67/94] feat: restore specific schemas --- borgmatic/actions/restore.py | 5 +++++ borgmatic/commands/arguments.py | 7 +++++++ borgmatic/hooks/postgresql.py | 2 ++ 3 files changed, 14 insertions(+) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index bbbb4c74..fa7b04b7 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -68,12 +68,15 @@ def restore_single_database( archive_name, hook_name, database, + schemas ): # pragma: no cover ''' Given (among other things) an archive name, a database hook name, and a configured database configuration dict, restore that database from the archive. ''' logger.info(f'{repository}: Restoring database {database["name"]}') + if schemas: + database['schemas'] = schemas dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', @@ -314,6 +317,7 @@ def run_restore( archive_name, found_hook_name or hook_name, found_database, + schemas = restore_arguments.schemas, ) # For any database that weren't found via exact matches in the hooks configuration, try to @@ -343,6 +347,7 @@ def run_restore( archive_name, found_hook_name or hook_name, database, + schemas = restore_arguments.schemas, ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 13ee3153..7f93910e 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -629,6 +629,13 @@ def make_parsers(): dest='databases', help="Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic's configuration", ) + restore_group.add_argument( + '--schema', + metavar='NAME', + nargs='+', + dest='schemas', + help="Names of schemas to restore from the database, defaults to all schemas." + ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' ) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index d4799f5f..5d951f6d 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -213,6 +213,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ('--command', 'ANALYZE') ) pg_restore_command = database.get('pg_restore_command') or 'pg_restore' + backup_schemas = ', '.join(database['schemas']) if 'schemas' in database else None restore_command = ( (psql_command if all_databases else pg_restore_command, '--no-password') + ( @@ -223,6 +224,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) + + (('--schema', backup_schemas) if backup_schemas else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) ) From 080c3afa0d43f3d8b2ffdaf23a869db71ccce03e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 5 Apr 2023 14:00:21 -0700 Subject: [PATCH 68/94] Fix documentation referring to "archive_name_format" in wrong configuration section. --- docs/how-to/make-per-application-backups.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 27c643f9..99f665bc 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -70,7 +70,7 @@ and borgmatic uses that format to name any new archive it creates. For instance: ```yaml -location: +storage: ... archive_name_format: home-directories-{now} ``` @@ -96,7 +96,7 @@ to filter archives when running supported actions. For instance, let's say that you have this in your configuration: ```yaml -location: +storage: ... archive_name_format: {hostname}-user-data-{now} ``` @@ -116,7 +116,7 @@ If this behavior isn't quite smart enough for your needs, you can use the filtering archives. For example: ```yaml -location: +storage: ... archive_name_format: {hostname}-user-data-{now} match_archives: sh:myhost-user-data-* From 192bfe46a95a39e6eb34d64dbff3d2dbcb9ecce1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 5 Apr 2023 14:58:05 -0700 Subject: [PATCH 69/94] Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set (#668). --- NEWS | 2 ++ borgmatic/borg/prune.py | 26 ++++++++++++++------------ tests/unit/borg/test_prune.py | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/NEWS b/NEWS index 96634fd0..f80169a8 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,8 @@ * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the "--match-archives" flag to correctly override the "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. + * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" + options set. 1.7.11 * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 7e12b9c2..3f06dc2d 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -26,22 +26,24 @@ def make_prune_flags(storage_config, retention_config, local_borg_version): config = retention_config.copy() prefix = config.pop('prefix', None) - if prefix: - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): - config['match_archives'] = f'sh:{prefix}*' - else: - config['glob_archives'] = f'{prefix}*' - flag_pairs = ( ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items() ) - return tuple( - element for pair in flag_pairs for element in pair - ) + flags.make_match_archives_flags( - storage_config.get('match_archives'), - storage_config.get('archive_name_format'), - local_borg_version, + return tuple(element for pair in flag_pairs for element in pair) + ( + ( + ('--match-archives', f'sh:{prefix}*') + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) + else ('--glob-archives', f'{prefix}*') + ) + if prefix + else ( + flags.make_match_archives_flags( + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, + ) + ) ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 37731aca..128bdc0a 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -69,6 +69,24 @@ def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives() assert result == expected +def test_make_prune_flags_prefers_prefix_to_archive_name_format(): + storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 + retention_config = OrderedDict((('keep_daily', 1), ('prefix', 'bar-'))) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').never() + + result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') + + expected = ( + '--keep-daily', + '1', + '--match-archives', + 'sh:bar-*', # noqa: FS003 + ) + + assert result == expected + + def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 retention_config = OrderedDict((('keep_daily', 1), ('prefix', None))) From 0fbdf8d860ec47257f23621397396ccfa563927a Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 6 Apr 2023 09:31:24 +0530 Subject: [PATCH 70/94] feat: add logfile name to hook context for interpolation --- borgmatic/commands/borgmatic.py | 6 ++++++ tests/unit/commands/test_borgmatic.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 22aba0b1..b6d3c4bb 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -269,6 +269,12 @@ def run_actions( 'repositories': ','.join([repo['path'] for repo in location['repositories']]), } + try: + log_file = global_arguments.log_file + hook_context['log_file'] = log_file + except AttributeError: + pass + command.execute_hook( hooks.get('before_actions'), hooks.get('umask'), diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 11e879df..44a08590 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -422,6 +422,26 @@ def test_run_actions_runs_rcreate(): ) ) +def test_run_actions_adds_log_file_to_hook_context(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.rcreate).should_receive('run_rcreate').once() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), + hooks={}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) def test_run_actions_runs_transfer(): flexmock(module).should_receive('add_custom_log_levels') From 091d60c226f1d66f98f21252d6b072b62080b57c Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 6 Apr 2023 12:36:10 +0530 Subject: [PATCH 71/94] refactor and improve tests --- borgmatic/commands/borgmatic.py | 6 +-- tests/unit/commands/test_borgmatic.py | 53 +++++++++++++++++---------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b6d3c4bb..a4de94ba 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -269,11 +269,7 @@ def run_actions( 'repositories': ','.join([repo['path'] for repo in location['repositories']]), } - try: - log_file = global_arguments.log_file - hook_context['log_file'] = log_file - except AttributeError: - pass + hook_context['log_file'] = global_arguments.log_file if global_arguments.log_file else '' command.execute_hook( hooks.get('before_actions'), diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 44a08590..63501853 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -408,7 +408,7 @@ def test_run_actions_runs_rcreate(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'rcreate': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -425,11 +425,24 @@ def test_run_actions_runs_rcreate(): def test_run_actions_adds_log_file_to_hook_context(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') - flexmock(borgmatic.actions.rcreate).should_receive('run_rcreate').once() + flexmock(borgmatic.actions.create).should_receive('run_create').with_args( + config_filename=flexmock(), + repository={'path': 'repo'}, + location={'repositories': []}, + storage=flexmock(), + hooks={}, + hook_context={'log_file': 'foo'}, + local_borg_version=flexmock(), + create_arguments=flexmock(), + global_arguments=flexmock(dry_run=False, log_file='foo'), + dry_run_label='', + local_path=flexmock(), + remote_path=flexmock(), + ).once() tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -450,7 +463,7 @@ def test_run_actions_runs_transfer(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'transfer': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'transfer': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -473,7 +486,7 @@ def test_run_actions_runs_create(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'create': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -496,7 +509,7 @@ def test_run_actions_runs_prune(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'prune': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'prune': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -518,7 +531,7 @@ def test_run_actions_runs_compact(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'compact': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'compact': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -541,7 +554,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'check': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -564,7 +577,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'check': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -586,7 +599,7 @@ def test_run_actions_runs_extract(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'extract': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'extract': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -608,7 +621,7 @@ def test_run_actions_runs_export_tar(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'export-tar': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'export-tar': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -630,7 +643,7 @@ def test_run_actions_runs_mount(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'mount': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'mount': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -652,7 +665,7 @@ def test_run_actions_runs_restore(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'restore': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'restore': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -675,7 +688,7 @@ def test_run_actions_runs_rlist(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'rlist': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rlist': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -699,7 +712,7 @@ def test_run_actions_runs_list(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'list': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'list': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -723,7 +736,7 @@ def test_run_actions_runs_rinfo(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'rinfo': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rinfo': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -747,7 +760,7 @@ def test_run_actions_runs_info(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'info': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'info': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -770,7 +783,7 @@ def test_run_actions_runs_break_lock(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'break-lock': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'break-lock': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -792,7 +805,7 @@ def test_run_actions_runs_borg(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'borg': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'borg': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -816,7 +829,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): tuple( module.run_actions( arguments={ - 'global': flexmock(dry_run=False), + 'global': flexmock(dry_run=False, log_file='foo'), 'borg': flexmock(), 'restore': flexmock(), }, From 16d7131fb75b2aef9bd02c0af47607c86c209572 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 7 Apr 2023 01:00:38 +0530 Subject: [PATCH 72/94] refactor tests --- borgmatic/commands/borgmatic.py | 3 +-- tests/unit/commands/test_borgmatic.py | 24 ++++++++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a4de94ba..5d56accb 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -267,10 +267,9 @@ def run_actions( 'repository': repository_path, # Deprecated: For backwards compatibility with borgmatic < 1.6.0. 'repositories': ','.join([repo['path'] for repo in location['repositories']]), + 'log_file': global_arguments.log_file if global_arguments.log_file else '', } - hook_context['log_file'] = global_arguments.log_file if global_arguments.log_file else '' - command.execute_hook( hooks.get('before_actions'), hooks.get('umask'), diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 63501853..56f2332c 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -422,25 +422,27 @@ def test_run_actions_runs_rcreate(): ) ) + def test_run_actions_adds_log_file_to_hook_context(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') + expected = flexmock() flexmock(borgmatic.actions.create).should_receive('run_create').with_args( - config_filename=flexmock(), + config_filename=object, repository={'path': 'repo'}, location={'repositories': []}, - storage=flexmock(), + storage=object, hooks={}, - hook_context={'log_file': 'foo'}, - local_borg_version=flexmock(), - create_arguments=flexmock(), - global_arguments=flexmock(dry_run=False, log_file='foo'), + hook_context={'repository': 'repo', 'repositories': '', 'log_file': 'foo'}, + local_borg_version=object, + create_arguments=object, + global_arguments=object, dry_run_label='', - local_path=flexmock(), - remote_path=flexmock(), - ).once() + local_path=object, + remote_path=object, + ).once().and_return(expected) - tuple( + result = tuple( module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), @@ -455,6 +457,8 @@ def test_run_actions_adds_log_file_to_hook_context(): repository={'path': 'repo'}, ) ) + assert result == (expected,) + def test_run_actions_runs_transfer(): flexmock(module).should_receive('add_custom_log_levels') From 4cd7556a34f1cff6c8c87047f18e6b6872567ab9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Apr 2023 13:58:37 -0700 Subject: [PATCH 73/94] Add "log_file" command hook context to NEWS and docs (#413). --- NEWS | 2 ++ docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/NEWS b/NEWS index f80169a8..1c5bb6f4 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.7.12.dev0 + * #413: Add "log_file" context to command hooks. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the "--match-archives" flag to correctly override the "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index c0bcfb89..426e1a85 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -66,6 +66,9 @@ variables you can use here: * `configuration_filename`: borgmatic configuration filename in which the hook was defined + * `log_file` + New in version 1.7.12: + path of the borgmatic log file, only set when the `--log-file` flag is used * `repository`: path of the current repository as configured in the current borgmatic configuration file From cc60a71210065767ffd598b57f79767be637fb05 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Apr 2023 14:12:12 -0700 Subject: [PATCH 74/94] Clarify "log_file" NEWS (#413). --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 1c5bb6f4..8c8f0ffc 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.7.12.dev0 - * #413: Add "log_file" context to command hooks. See the documentation for more information: + * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. + See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the "--match-archives" flag to correctly override the "match_archives" configuration option for From d6ef0df50d8b011de4d49ebc27fbe9962be0c893 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Apr 2023 10:01:08 -0700 Subject: [PATCH 75/94] Mention #670 being fixed in NEWS. --- NEWS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 8c8f0ffc..cd851826 100644 --- a/NEWS +++ b/NEWS @@ -2,9 +2,9 @@ * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ - * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the - "--match-archives" flag to correctly override the "match_archives" configuration option for - the "transfer", "list", "rlist", and "info" actions. + * #666, #670: Fix error when running the "info" action with the "--match-archives" or "--archive" + flags. Also fix the "--match-archives"/"--archive" flags to correctly override the + "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set. From 31a2ac914aa87d7d00c7b18ea04ec797bc96a261 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Apr 2023 14:26:54 -0700 Subject: [PATCH 76/94] Add optional support for running end-to-end tests and building documentation with rootless Podman instead of Docker. --- NEWS | 2 ++ docs/Dockerfile | 6 ++--- docs/how-to/develop-on-borgmatic.md | 36 ++++++++++++++++++++++++++++ scripts/run-end-to-end-dev-tests | 6 +++++ tests/end-to-end/docker-compose.yaml | 8 +++---- 5 files changed, 51 insertions(+), 7 deletions(-) diff --git a/NEWS b/NEWS index cd851826..81ebcf31 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,8 @@ "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set. + * Add optional support for running end-to-end tests and building documentation with rootless Podman + instead of Docker. 1.7.11 * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives diff --git a/docs/Dockerfile b/docs/Dockerfile index dcda44f3..b612596e 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.17.1 as borgmatic +FROM docker.io/alpine:3.17.1 as borgmatic COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib @@ -8,7 +8,7 @@ RUN borgmatic --help > /command-line.txt \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic "$action" --help >> /command-line.txt; done -FROM node:19.5.0-alpine as html +FROM docker.io/node:19.5.0-alpine as html ARG ENVIRONMENT=production @@ -28,7 +28,7 @@ COPY . /source RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \ && mv /output/docs/index.html /output/index.html -FROM nginx:1.22.1-alpine +FROM docker.io/nginx:1.22.1-alpine COPY --from=html /output /usr/share/nginx/html COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 6699bdf3..027a8145 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -101,6 +101,30 @@ scripts/run-end-to-end-dev-tests Note that this scripts assumes you have permission to run Docker. If you don't, then you may need to run with `sudo`. + +#### Podman + +New in version 1.7.12 +borgmatic's end-to-end tests optionally support using +[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md) +[Podman](https://podman.io/) instead of Docker. + +Setting up Podman is outside the scope of this documentation, but here are +some key points to double-check: + + * Install Podman along with `podman-docker` and your desired networking + support. + * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the + non-root user who will run tests. + * Create a non-root Podman socket for that user: + ```bash + systemctl --user enable --now podman.socket + ``` + +Then you'll be able to run end-to-end tests as per normal, and the test script +will automatically use your non-root Podman socket instead of a Docker socket. + + ## Code style Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply @@ -149,3 +173,15 @@ http://localhost:8080 to view the documentation with your changes. To close the documentation server, ctrl-C the script. Note that it does not currently auto-reload, so you'll need to stop it and re-run it for any additional documentation changes to take effect. + + +#### Podman + +New in version 1.7.12 +borgmatic's developer build for documentation optionally supports using +[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md) +[Podman](https://podman.io/) instead of Docker. + +Setting up Podman is outside the scope of this documentation. But once you +install `podman-docker`, then `scripts/dev-docs` should automatically use +Podman instead of Docker. diff --git a/scripts/run-end-to-end-dev-tests b/scripts/run-end-to-end-dev-tests index 575de00d..032967fd 100755 --- a/scripts/run-end-to-end-dev-tests +++ b/scripts/run-end-to-end-dev-tests @@ -10,5 +10,11 @@ set -e +USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock + +if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then + export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH" +fi + docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \ --renew-anon-volumes --abort-on-container-exit diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 80f12e9a..e6b0bf2b 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -1,22 +1,22 @@ version: '3' services: postgresql: - image: postgres:13.1-alpine + image: docker.io/postgres:13.1-alpine environment: POSTGRES_PASSWORD: test POSTGRES_DB: test mysql: - image: mariadb:10.5 + image: docker.io/mariadb:10.5 environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test mongodb: - image: mongo:5.0.5 + image: docker.io/mongo:5.0.5 environment: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: test tests: - image: alpine:3.13 + image: docker.io/alpine:3.13 environment: TEST_CONTAINER: true volumes: From 4c0e2cab78f2b13823a8f44f957911422770b960 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Apr 2023 10:49:09 -0700 Subject: [PATCH 77/94] View the results of configuration file merging via "validate-borgmatic-config --show" flag (#673). --- NEWS | 3 +++ borgmatic/commands/validate_config.py | 26 +++++++++++++++---- docs/how-to/make-per-application-backups.md | 17 ++++++++++++ docs/how-to/set-up-backups.md | 3 +++ tests/end-to-end/test_validate_config.py | 14 ++++++++++ .../commands/test_validate_config.py | 9 +++++++ 6 files changed, 67 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 81ebcf31..c9869b90 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,9 @@ "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set. + * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag. + See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes * Add optional support for running end-to-end tests and building documentation with rootless Podman instead of Docker. diff --git a/borgmatic/commands/validate_config.py b/borgmatic/commands/validate_config.py index 44c0082a..8aa8d321 100644 --- a/borgmatic/commands/validate_config.py +++ b/borgmatic/commands/validate_config.py @@ -2,6 +2,7 @@ import logging import sys from argparse import ArgumentParser +import borgmatic.config.generate from borgmatic.config import collect, validate logger = logging.getLogger(__name__) @@ -23,16 +24,22 @@ def parse_arguments(*arguments): default=config_paths, help=f'Configuration filenames or directories, defaults to: {config_paths}', ) + parser.add_argument( + '-s', + '--show', + action='store_true', + help='Show the validated configuration after all include merging has occurred', + ) return parser.parse_args(arguments) def main(): # pragma: no cover - args = parse_arguments(*sys.argv[1:]) + arguments = parse_arguments(*sys.argv[1:]) logging.basicConfig(level=logging.INFO, format='%(message)s') - config_filenames = tuple(collect.collect_config_filenames(args.config_paths)) + config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths)) if len(config_filenames) == 0: logger.critical('No files to validate found') sys.exit(1) @@ -40,13 +47,22 @@ def main(): # pragma: no cover found_issues = False for config_filename in config_filenames: try: - validate.parse_configuration(config_filename, validate.schema_filename()) + config, parse_logs = validate.parse_configuration( + config_filename, validate.schema_filename() + ) except (ValueError, OSError, validate.Validation_error) as error: logging.critical(f'{config_filename}: Error parsing configuration file') logging.critical(error) found_issues = True + else: + for log in parse_logs: + logger.handle(log) + + if arguments.show: + print('---') + print(borgmatic.config.generate.render_configuration(config)) if found_issues: sys.exit(1) - else: - logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}") + + logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}") diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 99f665bc..ab293472 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -276,6 +276,23 @@ include, the local file's option takes precedence. list values are appended together. +## Debugging includes + +New in version 1.7.12 If you'd +like to see what the loaded configuration looks like after includes get merged +in, run `validate-borgmatic-config` on your configuration file: + +```bash +sudo validate-borgmatic-config --show +``` + +You'll need to specify your configuration file with `--config` if it's not in +a default location. + +This will output the merged configuration as borgmatic sees it, which can be +helpful for understanding how your includes work in practice. + + ## Configuration overrides In more complex multi-application setups, you may want to override particular diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 01d79ff8..917eb437 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -180,6 +180,9 @@ following command is available for that: sudo validate-borgmatic-config ``` +You'll need to specify your configuration file with `--config` if it's not in +a default location. + This command's exit status (`$?` in Bash) is zero when configuration is valid and non-zero otherwise. diff --git a/tests/end-to-end/test_validate_config.py b/tests/end-to-end/test_validate_config.py index d41464e6..54465033 100644 --- a/tests/end-to-end/test_validate_config.py +++ b/tests/end-to-end/test_validate_config.py @@ -1,5 +1,6 @@ import os import subprocess +import sys import tempfile @@ -26,3 +27,16 @@ def test_validate_config_command_with_invalid_configuration_fails(): exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' ')) assert exit_code == 1 + + +def test_validate_config_command_with_show_flag_displays_configuration(): + with tempfile.TemporaryDirectory() as temporary_directory: + config_path = os.path.join(temporary_directory, 'test.yaml') + + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + output = subprocess.check_output( + f'validate-borgmatic-config --config {config_path} --show'.split(' ') + ).decode(sys.stdout.encoding) + + assert 'location:' in output + assert 'repositories:' in output diff --git a/tests/integration/commands/test_validate_config.py b/tests/integration/commands/test_validate_config.py index cb0144d2..78887e71 100644 --- a/tests/integration/commands/test_validate_config.py +++ b/tests/integration/commands/test_validate_config.py @@ -18,3 +18,12 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list(): parser = module.parse_arguments('--config', 'myconfig', 'otherconfig') assert parser.config_paths == ['myconfig', 'otherconfig'] + + +def test_parse_arguments_supports_show_flag(): + config_paths = ['default'] + flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) + + parser = module.parse_arguments('--config', 'myconfig', '--show') + + assert parser.show From 264cebd2b10cd13c3782d639bba32e07f556c2be Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 11 Apr 2023 23:19:49 +0530 Subject: [PATCH 78/94] complete psql multi schema backup --- borgmatic/actions/restore.py | 9 ++------- borgmatic/commands/arguments.py | 2 +- borgmatic/hooks/postgresql.py | 7 +++++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index fa7b04b7..1e8d175d 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -68,15 +68,12 @@ def restore_single_database( archive_name, hook_name, database, - schemas ): # pragma: no cover ''' Given (among other things) an archive name, a database hook name, and a configured database configuration dict, restore that database from the archive. ''' logger.info(f'{repository}: Restoring database {database["name"]}') - if schemas: - database['schemas'] = schemas dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', @@ -316,8 +313,7 @@ def run_restore( remote_path, archive_name, found_hook_name or hook_name, - found_database, - schemas = restore_arguments.schemas, + dict(found_database, **{'schemas': restore_arguments.schemas}), ) # For any database that weren't found via exact matches in the hooks configuration, try to @@ -346,8 +342,7 @@ def run_restore( remote_path, archive_name, found_hook_name or hook_name, - database, - schemas = restore_arguments.schemas, + dict(database, **{'schemas': restore_arguments.schemas}), ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 7f93910e..ebd7b8eb 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -634,7 +634,7 @@ def make_parsers(): metavar='NAME', nargs='+', dest='schemas', - help="Names of schemas to restore from the database, defaults to all schemas." + help="Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases", ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 5d951f6d..d296fa1c 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -213,7 +213,6 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ('--command', 'ANALYZE') ) pg_restore_command = database.get('pg_restore_command') or 'pg_restore' - backup_schemas = ', '.join(database['schemas']) if 'schemas' in database else None restore_command = ( (psql_command if all_databases else pg_restore_command, '--no-password') + ( @@ -224,10 +223,14 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) - + (('--schema', backup_schemas) if backup_schemas else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) ) + + if database['schemas']: + for schema in database['schemas']: + restore_command += ('--schema', schema) + extra_environment = make_extra_environment(database) logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") From 2fea429d785ba9902057472c2d3afc2cdb7ffa79 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 12 Apr 2023 09:34:19 +0530 Subject: [PATCH 79/94] collection restore for mongodb --- borgmatic/hooks/mongodb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index be5f656b..781e5f21 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -161,4 +161,7 @@ def build_restore_command(extract_process, database, dump_filename): command.extend(('--authenticationDatabase', database['authentication_database'])) if 'restore_options' in database: command.extend(database['restore_options'].split(' ')) + if database['schemas']: + for schema in database['schemas']: + command.extend(('--nsInclude', schema)) return command From 1ea4433aa98c6a15b6ba223f2835b900e54b5c55 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Apr 2023 21:49:10 -0700 Subject: [PATCH 80/94] Selectively shallow merge certain mappings or sequences when including configuration files (#672). --- NEWS | 3 + borgmatic/config/load.py | 79 +++-- docs/how-to/make-per-application-backups.md | 59 ++++ tests/integration/config/test_load.py | 303 +++++++++++++++++--- 4 files changed, 374 insertions(+), 70 deletions(-) diff --git a/NEWS b/NEWS index c9869b90..6d4f5448 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,9 @@ "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set. + * #672: Selectively shallow merge certain mappings or sequences when including configuration files. + See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index f3c45d52..e9caeb4c 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -38,6 +38,24 @@ def include_configuration(loader, filename_node, include_directory): return load_configuration(include_filename) +def retain_node_error(loader, node): + ''' + Given a ruamel.yaml.loader.Loader and a YAML node, raise an error. + + Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was + used in a configuration file without a merge. In configuration files with a merge, mapping and + sequence nodes with "!retain" tags are handled by deep_merge_nodes() below. + + Also raise ValueError if a scalar node is given, as "!retain" is not supported on scalar nodes. + ''' + if isinstance(node, (ruamel.yaml.nodes.MappingNode, ruamel.yaml.nodes.SequenceNode)): + raise ValueError( + 'The !retain tag may only be used within a configuration file containing a merged !include tag.' + ) + + raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.') + + class Include_constructor(ruamel.yaml.SafeConstructor): ''' A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including @@ -50,6 +68,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor): '!include', functools.partial(include_configuration, include_directory=include_directory), ) + self.add_constructor('!retain', retain_node_error) def flatten_mapping(self, node): ''' @@ -176,6 +195,8 @@ def deep_merge_nodes(nodes): ), ] + If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged. + The purpose of deep merging like this is to support, for instance, merging one borgmatic configuration file into another for reuse, such that a configuration section ("retention", etc.) does not completely replace the corresponding section in a merged file. @@ -198,32 +219,42 @@ def deep_merge_nodes(nodes): # If we're dealing with MappingNodes, recurse and merge its values as well. if isinstance(b_value, ruamel.yaml.nodes.MappingNode): - replaced_nodes[(b_key, b_value)] = ( - b_key, - ruamel.yaml.nodes.MappingNode( - tag=b_value.tag, - value=deep_merge_nodes(a_value.value + b_value.value), - start_mark=b_value.start_mark, - end_mark=b_value.end_mark, - flow_style=b_value.flow_style, - comment=b_value.comment, - anchor=b_value.anchor, - ), - ) + # A "!retain" tag says to skip deep merging for this node. Replace the tag so + # downstream schema validation doesn't break on our application-specific tag. + if b_value.tag == '!retain': + b_value.tag = 'tag:yaml.org,2002:map' + else: + replaced_nodes[(b_key, b_value)] = ( + b_key, + ruamel.yaml.nodes.MappingNode( + tag=b_value.tag, + value=deep_merge_nodes(a_value.value + b_value.value), + start_mark=b_value.start_mark, + end_mark=b_value.end_mark, + flow_style=b_value.flow_style, + comment=b_value.comment, + anchor=b_value.anchor, + ), + ) # If we're dealing with SequenceNodes, merge by appending one sequence to the other. elif isinstance(b_value, ruamel.yaml.nodes.SequenceNode): - replaced_nodes[(b_key, b_value)] = ( - b_key, - ruamel.yaml.nodes.SequenceNode( - tag=b_value.tag, - value=a_value.value + b_value.value, - start_mark=b_value.start_mark, - end_mark=b_value.end_mark, - flow_style=b_value.flow_style, - comment=b_value.comment, - anchor=b_value.anchor, - ), - ) + # A "!retain" tag says to skip deep merging for this node. Replace the tag so + # downstream schema validation doesn't break on our application-specific tag. + if b_value.tag == '!retain': + b_value.tag = 'tag:yaml.org,2002:seq' + else: + replaced_nodes[(b_key, b_value)] = ( + b_key, + ruamel.yaml.nodes.SequenceNode( + tag=b_value.tag, + value=a_value.value + b_value.value, + start_mark=b_value.start_mark, + end_mark=b_value.end_mark, + flow_style=b_value.flow_style, + comment=b_value.comment, + anchor=b_value.anchor, + ), + ) return [ replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index ab293472..ddaa37d2 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -276,6 +276,65 @@ include, the local file's option takes precedence. list values are appended together. +### Shallow merge + +Even though deep merging is generally pretty handy for included files, +sometimes you want specific sections in the local file to take precedence over +included sections—without any merging occuring for them. + +New in version 1.7.12 That's +where the `!retain` tag comes in. Whenever you're merging an included file +into your configuration file, you can optionally add the `!retain` tag to +particular local mappings or sequences to retain the local values and ignore +included values. + +For instance, start with this configuration file containing the `!retain` tag +on the `retention` mapping: + +```yaml +<<: !include /etc/borgmatic/common.yaml + +location: + repositories: + - repo.borg + +retention: !retain + keep_daily: 5 +``` + +And `common.yaml` like this: + +```yaml +location: + repositories: + - common.borg + +retention: + keep_hourly: 24 + keep_daily: 7 +``` + +Once this include gets merged in, the resulting configuration will have a +`keep_daily` value of `5` and nothing else in the `retention` section. That's +because the `!retain` tag says to retain the local version of `retention` and +ignore any values coming in from the include. But because the `repositories` +sequence doesn't have a `!retain` tag, that sequence still gets merged +together to contain both `common.borg` and `repo.borg`. + +The `!retain` tag can only be placed on mapping and sequence nodes, and it +goes right after the name of the option (and its colon) on the same line. The +effects of `!retain` are recursive, meaning that if you place a `!retain` tag +on a top-level mapping, even deeply nested values within it will not be +merged. + +Additionally, the `!retain` tag only works in a configuration file that also +performs a merge include with `<<: !include`. It doesn't make sense within, +for instance, an included configuration file itself (unless it in turn +performs its own merge include). That's because `!retain` only applies to the +file doing the include; it doesn't work in reverse or propagate through +includes. + + ## Debugging includes New in version 1.7.12 If you'd diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index 3382d734..ef9be0f8 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -2,7 +2,6 @@ import io import sys import pytest -import ruamel.yaml from flexmock import flexmock from borgmatic.config import load as module @@ -150,6 +149,99 @@ def test_load_configuration_merges_include(): assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'} +def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + foo: bar + baz: quux + + other: + a: b + c: d + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: !retain + foo: override + + other: + a: override + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + assert module.load_configuration('config.yaml') == { + 'stuff': {'foo': 'override'}, + 'other': {'a': 'override', 'c': 'd'}, + } + + +def test_load_configuration_with_retain_tag_but_without_merge_include_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: !retain + foo: bar + baz: quux + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + foo: override + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + assert module.load_configuration('config.yaml') + + +def test_load_configuration_with_retain_tag_on_scalar_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + foo: bar + baz: quux + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + foo: !retain override + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + assert module.load_configuration('config.yaml') + + def test_load_configuration_does_not_merge_include_list(): builtins = flexmock(sys.modules['builtins']) flexmock(module.os).should_receive('getcwd').and_return('/tmp') @@ -173,42 +265,59 @@ def test_load_configuration_does_not_merge_include_list(): config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) - with pytest.raises(ruamel.yaml.error.YAMLError): + with pytest.raises(module.ruamel.yaml.error.YAMLError): assert module.load_configuration('config.yaml') +@pytest.mark.parametrize( + 'node_class', + ( + module.ruamel.yaml.nodes.MappingNode, + module.ruamel.yaml.nodes.SequenceNode, + module.ruamel.yaml.nodes.ScalarNode, + ), +) +def test_retain_node_error_raises(node_class): + with pytest.raises(ValueError): + module.retain_node_error( + loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock()) + ) + + def test_deep_merge_nodes_replaces_colliding_scalar_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_hourly' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='24' + ), ), ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_daily' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), ), ], ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_daily' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), ), ], ), @@ -230,35 +339,39 @@ def test_deep_merge_nodes_replaces_colliding_scalar_values(): def test_deep_merge_nodes_keeps_non_colliding_scalar_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_hourly' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='24' + ), ), ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_daily' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), ), ], ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_minutely' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='10'), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='10' + ), ), ], ), @@ -282,28 +395,28 @@ def test_deep_merge_nodes_keeps_non_colliding_scalar_values(): def test_deep_merge_nodes_keeps_deeply_nested_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='lock_wait' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), ), ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='extra_borg_options' ), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='init' ), - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='--init-option' ), ), @@ -314,22 +427,22 @@ def test_deep_merge_nodes_keeps_deeply_nested_values(): ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='extra_borg_options' ), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='prune' ), - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='--prune-option' ), ), @@ -361,32 +474,32 @@ def test_deep_merge_nodes_keeps_deeply_nested_values(): def test_deep_merge_nodes_appends_colliding_sequence_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='before_backup' ), - ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:int', value=['echo 1', 'echo 2'] + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] ), ), ], ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='before_backup' ), - ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:int', value=['echo 3', 'echo 4'] + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', value=['echo 3', 'echo 4'] ), ), ], @@ -402,3 +515,101 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): assert len(options) == 1 assert options[0][0].value == 'before_backup' assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] + + +def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='keep_hourly' + ), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='24' + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='keep_daily' + ), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( + tag='!retain', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='keep_daily' + ), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), + ), + ], + ), + ), + ] + + result = module.deep_merge_nodes(node_values) + assert len(result) == 1 + (section_key, section_value) = result[0] + assert section_key.value == 'retention' + assert section_value.tag == 'tag:yaml.org,2002:map' + options = section_value.value + assert len(options) == 1 + assert options[0][0].value == 'keep_daily' + assert options[0][1].value == '5' + + +def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] + ), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='!retain', value=['echo 3', 'echo 4'] + ), + ), + ], + ), + ), + ] + + result = module.deep_merge_nodes(node_values) + assert len(result) == 1 + (section_key, section_value) = result[0] + assert section_key.value == 'hooks' + options = section_value.value + assert len(options) == 1 + assert options[0][0].value == 'before_backup' + assert options[0][1].tag == 'tag:yaml.org,2002:seq' + assert options[0][1].value == ['echo 3', 'echo 4'] From ea9213cb0384180455f1f21c1d416db159c90e72 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Apr 2023 22:12:57 -0700 Subject: [PATCH 81/94] Spelling. --- docs/how-to/make-per-application-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index ddaa37d2..9a3bfa33 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -280,7 +280,7 @@ list values are appended together. Even though deep merging is generally pretty handy for included files, sometimes you want specific sections in the local file to take precedence over -included sections—without any merging occuring for them. +included sections—without any merging occurring for them. New in version 1.7.12 That's where the `!retain` tag comes in. Whenever you're merging an included file From 08843d51d9946043dc47c717a9786b277952fb2c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Apr 2023 10:30:23 -0700 Subject: [PATCH 82/94] Replace "sequence" with "list" in docs for consistency. --- docs/how-to/make-per-application-backups.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 9a3bfa33..887bc431 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -285,7 +285,7 @@ included sections—without any merging occurring for them. New in version 1.7.12 That's where the `!retain` tag comes in. Whenever you're merging an included file into your configuration file, you can optionally add the `!retain` tag to -particular local mappings or sequences to retain the local values and ignore +particular local mappings or lists to retain the local values and ignore included values. For instance, start with this configuration file containing the `!retain` tag @@ -318,14 +318,13 @@ Once this include gets merged in, the resulting configuration will have a `keep_daily` value of `5` and nothing else in the `retention` section. That's because the `!retain` tag says to retain the local version of `retention` and ignore any values coming in from the include. But because the `repositories` -sequence doesn't have a `!retain` tag, that sequence still gets merged -together to contain both `common.borg` and `repo.borg`. +list doesn't have a `!retain` tag, it still gets merged together to contain +both `common.borg` and `repo.borg`. -The `!retain` tag can only be placed on mapping and sequence nodes, and it -goes right after the name of the option (and its colon) on the same line. The -effects of `!retain` are recursive, meaning that if you place a `!retain` tag -on a top-level mapping, even deeply nested values within it will not be -merged. +The `!retain` tag can only be placed on mappings and lists, and it goes right +after the name of the option (and its colon) on the same line. The effects of +`!retain` are recursive, meaning that if you place a `!retain` tag on a +top-level mapping, even deeply nested values within it will not be merged. Additionally, the `!retain` tag only works in a configuration file that also performs a merge include with `<<: !include`. It doesn't make sense within, From 4a94c2c9bff7e7e17db595086a92647e18a23d23 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 13 Apr 2023 14:39:36 -0700 Subject: [PATCH 83/94] Selectively omit list values when including configuration files (#672). --- NEWS | 3 + borgmatic/config/load.py | 32 ++- docs/how-to/make-per-application-backups.md | 60 ++++- tests/integration/config/test_load.py | 285 +++++++++++++++++++- 4 files changed, 362 insertions(+), 18 deletions(-) diff --git a/NEWS b/NEWS index 6d4f5448..a39e84d2 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,9 @@ * #672: Selectively shallow merge certain mappings or sequences when including configuration files. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge + * #672: Selectively omit list values when including configuration files. See the documentation for + more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#list-merge * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index e9caeb4c..068e661f 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -38,9 +38,9 @@ def include_configuration(loader, filename_node, include_directory): return load_configuration(include_filename) -def retain_node_error(loader, node): +def raise_retain_node_error(loader, node): ''' - Given a ruamel.yaml.loader.Loader and a YAML node, raise an error. + Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!retain" usage. Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was used in a configuration file without a merge. In configuration files with a merge, mapping and @@ -56,6 +56,19 @@ def retain_node_error(loader, node): raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.') +def raise_omit_node_error(loader, node): + ''' + Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!omit" usage. + + Raise ValueError unconditionally, as an "!omit" node here indicates it was used in a + configuration file without a merge. In configuration files with a merge, nodes with "!omit" + tags are handled by deep_merge_nodes() below. + ''' + raise ValueError( + 'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.' + ) + + class Include_constructor(ruamel.yaml.SafeConstructor): ''' A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including @@ -68,7 +81,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor): '!include', functools.partial(include_configuration, include_directory=include_directory), ) - self.add_constructor('!retain', retain_node_error) + self.add_constructor('!retain', raise_retain_node_error) + self.add_constructor('!omit', raise_omit_node_error) def flatten_mapping(self, node): ''' @@ -134,6 +148,16 @@ def load_configuration(filename): return config +def filter_omitted_nodes(nodes): + ''' + Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a + value matching such nodes. + ''' + omitted_values = tuple(node.value for node in nodes if node.tag == '!omit') + + return [node for node in nodes if node.value not in omitted_values] + + DELETED_NODE = object() @@ -247,7 +271,7 @@ def deep_merge_nodes(nodes): b_key, ruamel.yaml.nodes.SequenceNode( tag=b_value.tag, - value=a_value.value + b_value.value, + value=filter_omitted_nodes(a_value.value + b_value.value), start_mark=b_value.start_mark, end_mark=b_value.end_mark, flow_style=b_value.flow_style, diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 887bc431..9ee93f44 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -272,9 +272,65 @@ Once this include gets merged in, the resulting configuration would have a When there's an option collision between the local file and the merged include, the local file's option takes precedence. + +#### List merge + New in version 1.6.1 Colliding list values are appended together. +New in version 1.7.12 If there +is a list value from an include that you *don't* want in your local +configuration file, you can omit it with an `!omit` tag. For instance: + +```yaml +<<: !include /etc/borgmatic/common.yaml + +location: + source_directories: + - !omit /home + - /var +``` + +And `common.yaml` like this: + +```yaml +location: + source_directories: + - /home + - /etc +``` + +Once this include gets merged in, the resulting configuration will have a +`source_directories` value of `/etc` and `/var`—with `/home` omitted. + +This feature currently only works on scalar (e.g. string or number) list items +and will not work elsewhere in a configuration file. Be sure to put the +`!omit` tag *before* the list item (after the dash). Putting `!omit` after the +list item will not work, as it gets interpreted as part of the string. Here's +an example of some things not to do: + +```yaml +<<: !include /etc/borgmatic/common.yaml + +location: + source_directories: + # Do not do this! It will not work. "!omit" belongs before "/home". + - /home !omit + + # Do not do this either! "!omit" only works on scalar list items. + repositories: !omit + # Also do not do this for the same reason! This is a list item, but it's + # not a scalar. + - !omit path: repo.borg +``` + +Additionally, the `!omit` tag only works in a configuration file that also +performs a merge include with `<<: !include`. It doesn't make sense within, +for instance, an included configuration file itself (unless it in turn +performs its own merge include). That's because `!omit` only applies to the +file doing the include; it doesn't work in reverse or propagate through +includes. + ### Shallow merge @@ -296,7 +352,7 @@ on the `retention` mapping: location: repositories: - - repo.borg + - path: repo.borg retention: !retain keep_daily: 5 @@ -307,7 +363,7 @@ And `common.yaml` like this: ```yaml location: repositories: - - common.borg + - path: common.borg retention: keep_hourly: 24 diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index ef9be0f8..028a6523 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -211,7 +211,7 @@ def test_load_configuration_with_retain_tag_but_without_merge_include_raises(): builtins.should_receive('open').with_args('config.yaml').and_return(config_file) with pytest.raises(ValueError): - assert module.load_configuration('config.yaml') + module.load_configuration('config.yaml') def test_load_configuration_with_retain_tag_on_scalar_raises(): @@ -239,7 +239,156 @@ def test_load_configuration_with_retain_tag_on_scalar_raises(): builtins.should_receive('open').with_args('config.yaml').and_return(config_file) with pytest.raises(ValueError): - assert module.load_configuration('config.yaml') + module.load_configuration('config.yaml') + + +def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - x + - !omit b + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']} + + +def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - x + - !omit q + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']} + + +def test_load_configuration_with_omit_tag_on_non_list_item_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: !omit + - x + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') + + +def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - foo: bar + baz: quux + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - !omit foo: bar + baz: quux + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') + + +def test_load_configuration_with_omit_tag_but_without_merge_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - !omit b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - x + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') def test_load_configuration_does_not_merge_include_list(): @@ -277,13 +426,33 @@ def test_load_configuration_does_not_merge_include_list(): module.ruamel.yaml.nodes.ScalarNode, ), ) -def test_retain_node_error_raises(node_class): +def test_raise_retain_node_error_raises(node_class): with pytest.raises(ValueError): - module.retain_node_error( + module.raise_retain_node_error( loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock()) ) +def test_raise_omit_node_error_raises(): + with pytest.raises(ValueError): + module.raise_omit_node_error(loader=flexmock(), node=flexmock()) + + +def test_filter_omitted_nodes(): + nodes = [ + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'), + module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'), + ] + + result = module.filter_omitted_nodes(nodes) + + assert [item.value for item in result] == ['a', 'c', 'a', 'c'] + + def test_deep_merge_nodes_replaces_colliding_scalar_values(): node_values = [ ( @@ -483,7 +652,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): tag='tag:yaml.org,2002:str', value='before_backup' ), module.ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 1' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 2' + ), + ], ), ), ], @@ -499,7 +676,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): tag='tag:yaml.org,2002:str', value='before_backup' ), module.ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:seq', value=['echo 3', 'echo 4'] + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 4' + ), + ], ), ), ], @@ -514,10 +699,10 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): options = section_value.value assert len(options) == 1 assert options[0][0].value == 'before_backup' - assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] + assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] -def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain(): +def test_deep_merge_nodes_only_keeps_mapping_values_tagged_with_retain(): node_values = [ ( module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), @@ -568,7 +753,7 @@ def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain(): assert options[0][1].value == '5' -def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): +def test_deep_merge_nodes_only_keeps_sequence_values_tagged_with_retain(): node_values = [ ( module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), @@ -580,7 +765,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): tag='tag:yaml.org,2002:str', value='before_backup' ), module.ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 1' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 2' + ), + ], ), ), ], @@ -596,7 +789,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): tag='tag:yaml.org,2002:str', value='before_backup' ), module.ruamel.yaml.nodes.SequenceNode( - tag='!retain', value=['echo 3', 'echo 4'] + tag='!retain', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 4' + ), + ], ), ), ], @@ -612,4 +813,64 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): assert len(options) == 1 assert options[0][0].value == 'before_backup' assert options[0][1].tag == 'tag:yaml.org,2002:seq' - assert options[0][1].value == ['echo 3', 'echo 4'] + assert [item.value for item in options[0][1].value] == ['echo 3', 'echo 4'] + + +def test_deep_merge_nodes_skips_sequence_values_tagged_with_omit(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 1' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 2' + ), + ], + ), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode(tag='!omit', value='echo 2'), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + ], + ), + ), + ], + ), + ), + ] + + result = module.deep_merge_nodes(node_values) + assert len(result) == 1 + (section_key, section_value) = result[0] + assert section_key.value == 'hooks' + options = section_value.value + assert len(options) == 1 + assert options[0][0].value == 'before_backup' + assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 3'] From 613f6c602c19095538dbde585c275e40e1fe036d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 13 Apr 2023 15:12:19 -0700 Subject: [PATCH 84/94] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index a39e84d2..08003882 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.12.dev0 +1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ diff --git a/setup.py b/setup.py index 4b870eb4..92655fb9 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.12.dev0' +VERSION = '1.7.12' setup( From ebe5c5e83903685742a39ae63b779d7a64527cff Mon Sep 17 00:00:00 2001 From: polyzen Date: Fri, 14 Apr 2023 01:01:31 +0000 Subject: [PATCH 85/94] Fix "TypeError: 'module' object is not callable" in test_commands.py --- tests/integration/borg/test_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 49cd0e96..3f5f4445 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -1,6 +1,6 @@ import copy -import flexmock +from flexmock import flexmock import borgmatic.borg.info import borgmatic.borg.list @@ -105,4 +105,4 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): borgmatic.borg.info.display_archives_info( 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) - ) + ) \ No newline at end of file From 50b0a9ce383fcdd9847f03e1639bc263ab5b7c19 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 13 Apr 2023 19:13:50 -0700 Subject: [PATCH 86/94] Remove newline at end of file. --- tests/integration/borg/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 3f5f4445..1afb0e0f 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -105,4 +105,4 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): borgmatic.borg.info.display_archives_info( 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) - ) \ No newline at end of file + ) From f273e82d74d31f4706c3fcc721a5a97a185a539b Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 15 Apr 2023 02:57:51 +0530 Subject: [PATCH 87/94] add tests --- borgmatic/commands/arguments.py | 2 +- borgmatic/hooks/postgresql.py | 10 +++-- tests/unit/actions/test_restore.py | 32 ++++++++------ tests/unit/hooks/test_mongodb.py | 43 ++++++++++++++++--- tests/unit/hooks/test_postgresql.py | 66 +++++++++++++++++++++++++---- 5 files changed, 122 insertions(+), 31 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index ebd7b8eb..b89ca761 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -634,7 +634,7 @@ def make_parsers(): metavar='NAME', nargs='+', dest='schemas', - help="Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases", + help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases', ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index d296fa1c..a982209c 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -1,4 +1,5 @@ import csv +import itertools import logging import os @@ -225,12 +226,13 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (('--username', database['username']) if 'username' in database else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) + + tuple( + itertools.chain.from_iterable(('--schema', schema) for schema in database['schemas']) + if database['schemas'] + else () + ) ) - if database['schemas']: - for schema in database['schemas']: - restore_command += ('--schema', schema) - extra_environment = make_extra_environment(database) logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 8255c87d..667a23c3 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -233,7 +233,7 @@ def test_run_restore_restores_each_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -246,7 +246,7 @@ def test_run_restore_restores_each_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).once() flexmock(module).should_receive('ensure_databases_found') @@ -256,7 +256,9 @@ def test_run_restore_restores_each_database(): storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), @@ -327,7 +329,7 @@ def test_run_restore_restores_database_configured_with_all_name(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -340,7 +342,7 @@ def test_run_restore_restores_database_configured_with_all_name(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).once() flexmock(module).should_receive('ensure_databases_found') @@ -350,7 +352,9 @@ def test_run_restore_restores_database_configured_with_all_name(): storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), @@ -399,7 +403,7 @@ def test_run_restore_skips_missing_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -412,7 +416,7 @@ def test_run_restore_skips_missing_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).never() flexmock(module).should_receive('ensure_databases_found') @@ -422,7 +426,9 @@ def test_run_restore_skips_missing_database(): storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), @@ -465,7 +471,7 @@ def test_run_restore_restores_databases_from_different_hooks(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -478,7 +484,7 @@ def test_run_restore_restores_databases_from_different_hooks(): remote_path=object, archive_name=object, hook_name='mysql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).once() flexmock(module).should_receive('ensure_databases_found') @@ -488,7 +494,9 @@ def test_run_restore_restores_databases_from_different_hooks(): storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 44e427f1..77b830b7 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -157,7 +157,7 @@ def test_dump_databases_runs_mongodumpall_for_all_databases(): def test_restore_database_dump_runs_mongorestore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -189,7 +189,9 @@ def test_restore_database_dump_errors_on_multiple_database_config(): def test_restore_database_dump_runs_mongorestore_with_hostname_and_port(): - database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + database_config = [ + {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -223,6 +225,7 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): 'username': 'mongo', 'password': 'trustsome1', 'authentication_database': 'admin', + 'schemas': None, } ] extract_process = flexmock(stdout=flexmock()) @@ -254,7 +257,7 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): def test_restore_database_dump_runs_mongorestore_with_options(): - database_config = [{'name': 'foo', 'restore_options': '--harder'}] + database_config = [{'name': 'foo', 'restore_options': '--harder', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -271,8 +274,36 @@ def test_restore_database_dump_runs_mongorestore_with_options(): ) +def test_restore_databases_dump_runs_mongorestore_with_schemas(): + database_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + [ + 'mongorestore', + '--archive', + '--drop', + '--db', + 'foo', + '--nsInclude', + 'bar', + '--nsInclude', + 'baz', + ], + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + ) + + def test_restore_database_dump_runs_psql_for_all_database_dump(): - database_config = [{'name': 'all'}] + database_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -290,7 +321,7 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_with_dry_run_skips_restore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') @@ -302,7 +333,7 @@ def test_restore_database_dump_with_dry_run_skips_restore(): def test_restore_database_dump_without_extract_process_restores_from_disk(): - database_config = [{'name': 'foo', 'format': 'directory'}] + database_config = [{'name': 'foo', 'format': 'directory', 'schemas': None}] flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path') diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 349c04be..70cff920 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -411,7 +411,7 @@ def test_dump_databases_runs_non_default_pg_dump(): def test_restore_database_dump_runs_pg_restore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -458,7 +458,9 @@ def test_restore_database_dump_errors_on_multiple_database_config(): def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): - database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + database_config = [ + {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -506,7 +508,9 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): def test_restore_database_dump_runs_pg_restore_with_username_and_password(): - database_config = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}] + database_config = [ + {'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return( @@ -553,7 +557,12 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): def test_restore_database_dump_runs_pg_restore_with_options(): database_config = [ - {'name': 'foo', 'restore_options': '--harder', 'analyze_options': '--smarter'} + { + 'name': 'foo', + 'restore_options': '--harder', + 'analyze_options': '--smarter', + 'schemas': None, + } ] extract_process = flexmock(stdout=flexmock()) @@ -596,7 +605,7 @@ def test_restore_database_dump_runs_pg_restore_with_options(): def test_restore_database_dump_runs_psql_for_all_database_dump(): - database_config = [{'name': 'all'}] + database_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -621,7 +630,12 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): database_config = [ - {'name': 'foo', 'pg_restore_command': 'special_pg_restore', 'psql_command': 'special_psql'} + { + 'name': 'foo', + 'pg_restore_command': 'special_pg_restore', + 'psql_command': 'special_psql', + 'schemas': None, + } ] extract_process = flexmock(stdout=flexmock()) @@ -654,7 +668,7 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): def test_restore_database_dump_with_dry_run_skips_restore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') @@ -667,7 +681,7 @@ def test_restore_database_dump_with_dry_run_skips_restore(): def test_restore_database_dump_without_extract_process_restores_from_disk(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') @@ -696,3 +710,39 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): module.restore_database_dump( database_config, 'test.yaml', {}, dry_run=False, extract_process=None ) + + +def test_restore_database_dump_with_schemas_restores_schemas(): + database_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] + + flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'pg_restore', + '--no-password', + '--if-exists', + '--exit-on-error', + '--clean', + '--dbname', + 'foo', + '/dump/path', + '--schema', + 'bar', + '--schema', + 'baz', + ), + processes=[], + output_log_level=logging.DEBUG, + input_file=None, + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=None + ) From 96d4a8ee45c71109cb5091b445c2d72cf8525acf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 16:33:06 -0700 Subject: [PATCH 88/94] Add "borgmatic restore --schema" flag to NEWS (#375). --- NEWS | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 08003882..9743d77d 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.7.13.dev0 + * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" + flag. + 1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. See the documentation for more information: diff --git a/setup.py b/setup.py index 92655fb9..f6fca3db 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.12' +VERSION = '1.7.13.dev0' setup( From 1c67db5d627a9f936297747e3c410e868f625450 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 16:40:58 -0700 Subject: [PATCH 89/94] Add documentation for "borgmatic restore --schema" (#375). --- NEWS | 3 ++- docs/how-to/backup-your-databases.md | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 9743d77d..4a27d074 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.7.13.dev0 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" - flag. + flag. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas 1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 0b46e4c1..91dba18f 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -324,6 +324,17 @@ includes any combined dump file named "all" and any other individual database dumps found in the archive. +### Restore particular schemas + +New in version 1.7.13 With +PostgreSQL and MongoDB, you can limit the restore to a single schema found +within the database dump: + +```bash +borgmatic restore --archive latest --database users --schema tentant1 +``` + + ### Limitations There are a few important limitations with borgmatic's current database From 5dbb71709c505c7a511209aeab1527af43591487 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 19:35:24 -0700 Subject: [PATCH 90/94] Upgrade test requirements and code style requirements. Auto-reformat code accordingly. --- borgmatic/actions/borg.py | 7 +- borgmatic/actions/break_lock.py | 7 +- borgmatic/actions/info.py | 7 +- borgmatic/actions/list.py | 7 +- borgmatic/actions/mount.py | 7 +- borgmatic/actions/restore.py | 10 ++- borgmatic/actions/rinfo.py | 7 +- borgmatic/actions/rlist.py | 7 +- borgmatic/borg/break_lock.py | 6 +- borgmatic/borg/create.py | 4 +- borgmatic/borg/export_tar.py | 6 +- borgmatic/borg/extract.py | 6 +- borgmatic/borg/info.py | 3 +- borgmatic/borg/rinfo.py | 3 +- borgmatic/borg/rlist.py | 3 +- borgmatic/borg/version.py | 3 +- borgmatic/commands/arguments.py | 14 ++-- borgmatic/commands/borgmatic.py | 48 ++++++++++--- borgmatic/config/load.py | 1 + borgmatic/config/normalize.py | 10 ++- borgmatic/config/override.py | 9 ++- borgmatic/execute.py | 6 +- borgmatic/hooks/mysql.py | 4 +- borgmatic/hooks/postgresql.py | 11 ++- setup.cfg | 11 +-- test_requirements.txt | 36 +++++----- tests/unit/actions/test_check.py | 6 +- tests/unit/actions/test_restore.py | 10 ++- tests/unit/borg/test_borg.py | 5 +- tests/unit/borg/test_break_lock.py | 29 ++++++-- tests/unit/borg/test_check.py | 62 ++++++++++++++--- tests/unit/borg/test_create.py | 20 ++++-- tests/unit/borg/test_extract.py | 14 +++- tests/unit/borg/test_flags.py | 27 ++++++-- tests/unit/borg/test_info.py | 9 ++- tests/unit/borg/test_list.py | 28 ++++++-- tests/unit/borg/test_mount.py | 11 ++- tests/unit/borg/test_rcreate.py | 98 +++++++++++++++++++++++---- tests/unit/borg/test_rinfo.py | 72 ++++++++++++++++---- tests/unit/borg/test_rlist.py | 29 ++++++-- tests/unit/borg/test_version.py | 3 +- tests/unit/commands/test_borgmatic.py | 3 +- tests/unit/config/test_normalize.py | 40 ++++++++--- tests/unit/config/test_validate.py | 7 +- tests/unit/hooks/test_cronhub.py | 6 +- tests/unit/hooks/test_cronitor.py | 6 +- tests/unit/hooks/test_healthchecks.py | 4 +- tests/unit/hooks/test_mongodb.py | 3 +- tests/unit/hooks/test_mysql.py | 17 ++++- tests/unit/test_execute.py | 6 +- 50 files changed, 603 insertions(+), 155 deletions(-) diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index fbf1d49d..3d2998b6 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_borg( - repository, storage, local_borg_version, borg_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + borg_arguments, + local_path, + remote_path, ): ''' Run the "borg" action for the given repository. diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index eb6e4547..2174161c 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -7,7 +7,12 @@ logger = logging.getLogger(__name__) def run_break_lock( - repository, storage, local_borg_version, break_lock_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + break_lock_arguments, + local_path, + remote_path, ): ''' Run the "break-lock" action for the given repository. diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index 6cd775f5..54023127 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -9,7 +9,12 @@ logger = logging.getLogger(__name__) def run_info( - repository, storage, local_borg_version, info_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + info_arguments, + local_path, + remote_path, ): ''' Run the "info" action for the given repository and archive. diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 0e736f84..359f3b67 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_list( - repository, storage, local_borg_version, list_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + list_arguments, + local_path, + remote_path, ): ''' Run the "list" action for the given repository and archive. diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index 6e787b18..72e321a0 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_mount( - repository, storage, local_borg_version, mount_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + mount_arguments, + local_path, + remote_path, ): ''' Run the "mount" action for the given repository. diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 1e8d175d..f061dca8 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -114,7 +114,13 @@ def restore_single_database( def collect_archive_database_names( - repository, archive, location, storage, local_borg_version, local_path, remote_path, + repository, + archive, + location, + storage, + local_borg_version, + local_path, + remote_path, ): ''' Given a local or remote repository path, a resolved archive name, a location configuration dict, @@ -180,7 +186,7 @@ def find_databases_to_restore(requested_database_names, archive_database_names): if 'all' in restore_names[UNSPECIFIED_HOOK]: restore_names[UNSPECIFIED_HOOK].remove('all') - for (hook_name, database_names) in archive_database_names.items(): + for hook_name, database_names in archive_database_names.items(): restore_names.setdefault(hook_name, []).extend(database_names) # If a database is to be restored as part of "all", then remove it from restore names so diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 001a0851..0947ec3d 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_rinfo( - repository, storage, local_borg_version, rinfo_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + rinfo_arguments, + local_path, + remote_path, ): ''' Run the "rinfo" action for the given repository. diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index c2ea1c63..10d06a51 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_rlist( - repository, storage, local_borg_version, rlist_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ): ''' Run the "rlist" action for the given repository. diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 2dff31ec..7099af83 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -7,7 +7,11 @@ logger = logging.getLogger(__name__) def break_lock( - repository_path, storage_config, local_borg_version, local_path='borg', remote_path=None, + repository_path, + storage_config, + local_borg_version, + local_path='borg', + remote_path=None, ): ''' Given a local or remote repository path, a storage configuration dict, the local Borg version, diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 58c3509b..8782dc6b 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -509,7 +509,9 @@ def create_archive( ) elif output_log_level is None: return execute_command_and_capture_output( - create_command, working_directory=working_directory, extra_environment=borg_environment, + create_command, + working_directory=working_directory, + extra_environment=borg_environment, ) else: execute_command( diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 2b427f03..a624f07d 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -45,7 +45,11 @@ def export_tar_archive( + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + flags.make_repository_archive_flags(repository_path, archive, local_borg_version,) + + flags.make_repository_archive_flags( + repository_path, + archive, + local_borg_version, + ) + (destination_path,) + (tuple(paths) if paths else ()) ) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index a95685cf..f9471416 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -108,7 +108,11 @@ def extract_archive( + (('--strip-components', str(strip_components)) if strip_components else ()) + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) - + flags.make_repository_archive_flags(repository, archive, local_borg_version,) + + flags.make_repository_archive_flags( + repository, + archive, + local_borg_version, + ) + (tuple(paths) if paths else ()) ) diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index 80739c07..ef2c0c44 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -62,7 +62,8 @@ def display_archives_info( if info_arguments.json: return execute_command_and_capture_output( - full_command, extra_environment=environment.make_environment(storage_config), + full_command, + extra_environment=environment.make_environment(storage_config), ) else: execute_command( diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index 8a30b7f0..97d7a666 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -50,7 +50,8 @@ def display_repository_info( if rinfo_arguments.json: return execute_command_and_capture_output( - full_command, extra_environment=extra_environment, + full_command, + extra_environment=extra_environment, ) else: execute_command( diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 9686b67b..c051a9ad 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -40,7 +40,8 @@ def resolve_archive_name( ) output = execute_command_and_capture_output( - full_command, extra_environment=environment.make_environment(storage_config), + full_command, + extra_environment=environment.make_environment(storage_config), ) try: latest_archive = output.strip().splitlines()[-1] diff --git a/borgmatic/borg/version.py b/borgmatic/borg/version.py index 6d6c302c..d90a7aae 100644 --- a/borgmatic/borg/version.py +++ b/borgmatic/borg/version.py @@ -19,7 +19,8 @@ def local_borg_version(storage_config, local_path='borg'): + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) ) output = execute_command_and_capture_output( - full_command, extra_environment=environment.make_environment(storage_config), + full_command, + extra_environment=environment.make_environment(storage_config), ) try: diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index b89ca761..61b54769 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -178,7 +178,9 @@ def make_parsers(): help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)', ) global_group.add_argument( - '--log-file', type=str, help='Write log messages to this file instead of syslog', + '--log-file', + type=str, + help='Write log messages to this file instead of syslog', ) global_group.add_argument( '--log-file-format', @@ -258,10 +260,13 @@ def make_parsers(): help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key (Borg 2.x+ only)', ) rcreate_group.add_argument( - '--append-only', action='store_true', help='Create an append-only repository', + '--append-only', + action='store_true', + help='Create an append-only repository', ) rcreate_group.add_argument( - '--storage-quota', help='Create a repository with a fixed storage quota', + '--storage-quota', + help='Create a repository with a fixed storage quota', ) rcreate_group.add_argument( '--make-parent-dirs', @@ -649,7 +654,8 @@ def make_parsers(): ) rlist_group = rlist_parser.add_argument_group('rlist arguments') rlist_group.add_argument( - '--repository', help='Path of repository to list, defaults to the configured repositories', + '--repository', + help='Path of repository to list, defaults to the configured repositories', ) rlist_group.add_argument( '--short', default=False, action='store_true', help='Output only archive names' diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5d56accb..fece475a 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -103,7 +103,9 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: repo_queue = Queue() for repo in location['repositories']: - repo_queue.put((repo, 0),) + repo_queue.put( + (repo, 0), + ) while not repo_queue.empty(): repository, retry_num = repo_queue.get() @@ -128,7 +130,9 @@ def run_configuration(config_filename, config, arguments): ) except (OSError, CalledProcessError, ValueError) as error: if retry_num < retries: - repo_queue.put((repository, retry_num + 1),) + repo_queue.put( + (repository, retry_num + 1), + ) tuple( # Consume the generator so as to trigger logging. log_error_records( f'{repository["path"]}: Error running actions for repository', @@ -279,7 +283,7 @@ def run_actions( **hook_context, ) - for (action_name, action_arguments) in arguments.items(): + for action_name, action_arguments in arguments.items(): if action_name == 'rcreate': borgmatic.actions.rcreate.run_rcreate( repository, @@ -408,19 +412,39 @@ def run_actions( ) elif action_name == 'rlist': yield from borgmatic.actions.rlist.run_rlist( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'list': yield from borgmatic.actions.list.run_list( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'rinfo': yield from borgmatic.actions.rinfo.run_rinfo( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'info': yield from borgmatic.actions.info.run_info( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'break-lock': borgmatic.actions.break_lock.run_break_lock( @@ -433,7 +457,12 @@ def run_actions( ) elif action_name == 'borg': borgmatic.actions.borg.run_borg( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) command.execute_hook( @@ -626,7 +655,8 @@ def collect_configuration_run_summary_logs(configs, arguments): logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") try: borg_umount.unmount_archive( - mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs), + mount_point=arguments['umount'].mount_point, + local_path=get_local_path(configs), ) except (CalledProcessError, OSError) as error: yield from log_error_records('Error unmounting mount point', error) diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 068e661f..f6290de8 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -121,6 +121,7 @@ def load_configuration(filename): Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError if there are too many recursive includes. ''' + # Use an embedded derived class for the include constructor so as to capture the filename # value. (functools.partial doesn't work for this use case because yaml.Constructor has to be # an actual class.) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index bcf088ac..147e4e4e 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -81,7 +81,10 @@ def normalize(config_filename, config): repository_path.partition('file://')[-1] ) config['location']['repositories'].append( - dict(repository_dict, path=updated_repository_path,) + dict( + repository_dict, + path=updated_repository_path, + ) ) elif repository_path.startswith('ssh://'): config['location']['repositories'].append(repository_dict) @@ -97,7 +100,10 @@ def normalize(config_filename, config): ) ) config['location']['repositories'].append( - dict(repository_dict, path=rewritten_repository_path,) + dict( + repository_dict, + path=rewritten_repository_path, + ) ) else: config['location']['repositories'].append(repository_dict) diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 8b2a1ab8..aacf375d 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -57,7 +57,12 @@ def parse_overrides(raw_overrides): for raw_override in raw_overrides: try: raw_keys, value = raw_override.split('=', 1) - parsed_overrides.append((tuple(raw_keys.split('.')), convert_value_type(value),)) + parsed_overrides.append( + ( + tuple(raw_keys.split('.')), + convert_value_type(value), + ) + ) except ValueError: raise ValueError( f"Invalid override '{raw_override}'. Make sure you use the form: SECTION.OPTION=VALUE" @@ -75,5 +80,5 @@ def apply_overrides(config, raw_overrides): ''' overrides = parse_overrides(raw_overrides) - for (keys, value) in overrides: + for keys, value in overrides: set_values(config, keys, value) diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 0afa5cca..39691dac 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -236,7 +236,11 @@ def execute_command( def execute_command_and_capture_output( - full_command, capture_stderr=False, shell=False, extra_environment=None, working_directory=None, + full_command, + capture_stderr=False, + shell=False, + extra_environment=None, + working_directory=None, ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index aeeeee5b..793b78b0 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -100,7 +100,9 @@ def execute_dump_command( dump.create_named_pipe_for_dump(dump_filename) return execute_command( - dump_command, extra_environment=extra_environment, run_to_completion=False, + dump_command, + extra_environment=extra_environment, + run_to_completion=False, ) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index a982209c..bcc48efd 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -123,7 +123,12 @@ def dump_databases(databases, log_prefix, location_config, dry_run): continue command = ( - (dump_command, '--no-password', '--clean', '--if-exists',) + ( + dump_command, + '--no-password', + '--clean', + '--if-exists', + ) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -146,7 +151,9 @@ def dump_databases(databases, log_prefix, location_config, dry_run): if dump_format == 'directory': dump.create_parent_directory_for_dump(dump_filename) execute_command( - command, shell=True, extra_environment=extra_environment, + command, + shell=True, + extra_environment=extra_environment, ) else: dump.create_named_pipe_for_dump(dump_filename) diff --git a/setup.cfg b/setup.cfg index f97ae972..30841386 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,18 +8,21 @@ filterwarnings = ignore:Deprecated call to `pkg_resources.declare_namespace\('ruamel'\)`.*:DeprecationWarning [flake8] -ignore = E501,W503 +max-line-length = 100 +extend-ignore = E203,E501,W503 exclude = *.*/* multiline-quotes = ''' docstring-quotes = ''' [tool:isort] -force_single_line = False -include_trailing_comma = True +profile=black known_first_party = borgmatic line_length = 100 -multi_line_output = 3 skip = .tox [codespell] skip = .git,.tox,build + +[pycodestyle] +ignore = E203 +max_line_length = 100 diff --git a/test_requirements.txt b/test_requirements.txt index 7910788b..67e02563 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,27 +1,29 @@ appdirs==1.4.4; python_version >= '3.8' -attrs==20.3.0; python_version >= '3.8' -black==19.10b0; python_version >= '3.8' -click==7.1.2; python_version >= '3.8' +attrs==22.2.0; python_version >= '3.8' +black==23.3.0; python_version >= '3.8' +chardet==5.1.0 +click==8.1.3; python_version >= '3.8' codespell==2.2.4 -colorama==0.4.4 -coverage==5.3 -flake8==4.0.1 +colorama==0.4.6 +coverage==7.2.3 +flake8==6.0.0 flake8-quotes==3.3.2 flake8-use-fstring==1.4 flake8-variables-names==0.0.5 -flexmock==0.10.4 -isort==5.9.1 -mccabe==0.6.1 -pluggy==0.13.1 -pathspec==0.8.1; python_version >= '3.8' -py==1.10.0 -pycodestyle==2.8.0 -pyflakes==2.4.0 -jsonschema==3.2.0 -pytest==7.2.0 +flexmock==0.11.3 +idna==3.4 +isort==5.12.0 +mccabe==0.7.0 +pluggy==1.0.0 +pathspec==0.11.1; python_version >= '3.8' +py==1.11.0 +pycodestyle==2.10.0 +pyflakes==3.0.1 +jsonschema==4.17.3 +pytest==7.3.0 pytest-cov==4.0.0 regex; python_version >= '3.8' -requests==2.25.0 +requests==2.28.2 ruamel.yaml>0.15.0,<0.18.0 toml==0.10.2; python_version >= '3.8' typed-ast; python_version >= '3.8' diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 4c2027eb..05f63b6a 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -12,7 +12,11 @@ def test_run_check_calls_hooks_for_configured_repository(): flexmock(module.borgmatic.borg.check).should_receive('check_archives').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( - repository=None, progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock(), + repository=None, + progress=flexmock(), + repair=flexmock(), + only=flexmock(), + force=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 667a23c3..16fe2920 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -148,7 +148,8 @@ def test_find_databases_to_restore_without_requested_names_finds_all_archive_dat archive_database_names = {'postresql_databases': ['foo', 'bar']} restore_names = module.find_databases_to_restore( - requested_database_names=[], archive_database_names=archive_database_names, + requested_database_names=[], + archive_database_names=archive_database_names, ) assert restore_names == archive_database_names @@ -158,7 +159,8 @@ def test_find_databases_to_restore_with_all_in_requested_names_finds_all_archive archive_database_names = {'postresql_databases': ['foo', 'bar']} restore_names = module.find_databases_to_restore( - requested_database_names=['all'], archive_database_names=archive_database_names, + requested_database_names=['all'], + archive_database_names=archive_database_names, ) assert restore_names == archive_database_names @@ -194,7 +196,9 @@ def test_ensure_databases_found_with_all_databases_found_does_not_raise(): def test_ensure_databases_found_with_no_databases_raises(): with pytest.raises(ValueError): module.ensure_databases_found( - restore_names={'postgresql_databases': []}, remaining_restore_names={}, found_names=[], + restore_names={'postgresql_databases': []}, + remaining_restore_names={}, + found_names=[], ) diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 545da6c0..5b735960 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -222,7 +222,10 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): ) module.run_arbitrary_borg( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=[], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=[], ) diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 210aceeb..509fc1b8 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -10,7 +10,9 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock(command): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - command, borg_local_path='borg', extra_environment=None, + command, + borg_local_path='borg', + extra_environment=None, ).once() @@ -19,7 +21,9 @@ def test_break_lock_calls_borg_with_required_flags(): insert_execute_command_mock(('borg', 'break-lock', 'repo')) module.break_lock( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', ) @@ -28,7 +32,10 @@ def test_break_lock_calls_borg_with_remote_path_flags(): insert_execute_command_mock(('borg', 'break-lock', '--remote-path', 'borg1', 'repo')) module.break_lock( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + remote_path='borg1', ) @@ -37,7 +44,9 @@ def test_break_lock_calls_borg_with_umask_flags(): insert_execute_command_mock(('borg', 'break-lock', '--umask', '0770', 'repo')) module.break_lock( - repository_path='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={'umask': '0770'}, + local_borg_version='1.2.3', ) @@ -46,7 +55,9 @@ def test_break_lock_calls_borg_with_lock_wait_flags(): insert_execute_command_mock(('borg', 'break-lock', '--lock-wait', '5', 'repo')) module.break_lock( - repository_path='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={'lock_wait': '5'}, + local_borg_version='1.2.3', ) @@ -56,7 +67,9 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.break_lock( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', ) @@ -66,5 +79,7 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags(): insert_logging_mock(logging.DEBUG) module.break_lock( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 65e17291..1f992d3d 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -79,7 +79,12 @@ def test_parse_frequency_parses_into_timedeltas(frequency, expected_result): @pytest.mark.parametrize( - 'frequency', ('sometime', 'x days', '3 decades',), + 'frequency', + ( + 'sometime', + 'x days', + '3 decades', + ), ) def test_parse_frequency_raises_on_parse_error(frequency): with pytest.raises(ValueError): @@ -211,7 +216,10 @@ def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flags = module.make_check_flags('1.2.3', {}, ('data',)) - assert flags == ('--archives-only', '--verify-data',) + assert flags == ( + '--archives-only', + '--verify-data', + ) def test_make_check_flags_with_extract_omits_extract_flag(): @@ -227,7 +235,14 @@ def test_make_check_flags_with_repository_and_data_checks_does_not_return_reposi flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('repository', 'data',)) + flags = module.make_check_flags( + '1.2.3', + {}, + ( + 'repository', + 'data', + ), + ) assert flags == ('--verify-data',) @@ -236,7 +251,12 @@ def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags() flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo',) + flags = module.make_check_flags( + '1.2.3', + {}, + ('repository', 'archives'), + prefix='foo', + ) assert flags == ('--match-archives', 'sh:foo*') @@ -246,7 +266,10 @@ def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags(): flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', ) assert flags == ('--match-archives', 'sh:foo*') @@ -257,7 +280,10 @@ def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_retur flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', ) assert flags == ('--glob-archives', 'foo*') @@ -447,7 +473,11 @@ def test_check_archives_calls_borg_with_parameters(checks): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', {}, checks, check_last, prefix=None, + '1.2.3', + {}, + checks, + check_last, + prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) @@ -601,7 +631,11 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', {}, checks, check_last, prefix=None, + '1.2.3', + {}, + checks, + check_last, + prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1', 'check', 'repo')) @@ -628,7 +662,11 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', {}, checks, check_last, prefix=None, + '1.2.3', + {}, + checks, + check_last, + prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo')) @@ -656,7 +694,11 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', storage_config, checks, check_last, None, + '1.2.3', + storage_config, + checks, + check_last, + None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo')) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 42fedd56..818ce276 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -1053,7 +1053,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), + 'feature_available,option_flag', + ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), ) def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_parameters( feature_available, option_flag @@ -1188,7 +1189,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner')), + 'feature_available,option_flag', + ((True, '--numeric-ids'), (False, '--numeric-owner')), ) def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( feature_available, option_flag @@ -1290,7 +1292,12 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter @pytest.mark.parametrize( 'option_name,option_value', - (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),), + ( + ('ctime', True), + ('ctime', False), + ('birthtime', True), + ('birthtime', False), + ), ) def test_create_archive_with_basic_option_calls_borg_with_corresponding_parameter( option_name, option_value @@ -1766,7 +1773,12 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--info', '--progress',), + ('borg', 'create') + + REPO_ARCHIVE_WITH_PATHS + + ( + '--info', + '--progress', + ), output_log_level=logging.INFO, output_file=module.DO_NOT_CAPTURE, borg_local_path='borg', diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index de29cf4d..26fd7380 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -11,7 +11,9 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock(command, working_directory=None): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - command, working_directory=working_directory, extra_environment=None, + command, + working_directory=working_directory, + extra_environment=None, ).once() @@ -152,7 +154,11 @@ def test_extract_archive_calls_borg_with_remote_path_parameters(): @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner'),), + 'feature_available,option_flag', + ( + (True, '--numeric-ids'), + (False, '--numeric-owner'), + ), ) def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available, option_flag): flexmock(module.os.path).should_receive('abspath').and_return('repo') @@ -441,7 +447,9 @@ def test_extract_archive_skips_abspath_for_remote_repository(): flexmock(module.os.path).should_receive('abspath').never() flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'extract', 'server:repo::archive'), working_directory=None, extra_environment=None, + ('borg', 'extract', 'server:repo::archive'), + working_directory=None, + extra_environment=None, ).once() flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 86247040..2eaff0ae 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -70,7 +70,11 @@ def test_make_repository_archive_flags_with_borg_features_separates_repository_a assert module.make_repository_archive_flags( repository_path='repo', archive='archive', local_borg_version='1.2.3' - ) == ('--repo', 'repo', 'archive',) + ) == ( + '--repo', + 'repo', + 'archive', + ) def test_make_repository_archive_flags_with_borg_features_joins_repository_and_archive(): @@ -86,9 +90,24 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a ( (None, None, True, ()), (None, '', True, ()), - ('re:foo-.*', '{hostname}-{now}', True, ('--match-archives', 're:foo-.*'),), # noqa: FS003 - ('sh:foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003 - ('foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003 + ( + 're:foo-.*', + '{hostname}-{now}', + True, + ('--match-archives', 're:foo-.*'), + ), # noqa: FS003 + ( + 'sh:foo-*', + '{hostname}-{now}', + False, + ('--glob-archives', 'foo-*'), + ), # noqa: FS003 + ( + 'foo-*', + '{hostname}-{now}', + False, + ('--glob-archives', 'foo-*'), + ), # noqa: FS003 ( None, '{hostname}-docs-{now}', # noqa: FS003 diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index d51bf14d..2eed4fea 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -69,7 +69,8 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'info', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.INFO) @@ -120,7 +121,8 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'info', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.DEBUG) @@ -145,7 +147,8 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'info', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') json_output = module.display_archives_info( diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 37af65a3..0a7db4cc 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -387,7 +387,8 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module.rlist).should_receive('make_rlist_command').and_return(('borg', 'list', 'repo')) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list', 'repo'), extra_environment=None, + ('borg', 'list', 'repo'), + extra_environment=None, ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').and_return( ('borg', 'list', 'repo::archive1') @@ -518,9 +519,18 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos @pytest.mark.parametrize( - 'archive_filter_flag', ('prefix', 'match_archives', 'sort_by', 'first', 'last',), + 'archive_filter_flag', + ( + 'prefix', + 'match_archives', + 'sort_by', + 'first', + 'last', + ), ) -def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_flag,): +def test_list_archive_with_archive_ignores_archive_filter_flag( + archive_filter_flag, +): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.logger).answer = lambda message: None @@ -566,7 +576,14 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_fl @pytest.mark.parametrize( - 'archive_filter_flag', ('prefix', 'match_archives', 'sort_by', 'first', 'last',), + 'archive_filter_flag', + ( + 'prefix', + 'match_archives', + 'sort_by', + 'first', + 'last', + ), ) def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes_it_to_rlist( archive_filter_flag, @@ -597,7 +614,8 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).and_return(('borg', 'rlist', '--repo', 'repo')) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rlist', '--repo', 'repo'), extra_environment=None, + ('borg', 'rlist', '--repo', 'repo'), + extra_environment=None, ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').with_args( diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 6161a249..658b2e52 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -10,7 +10,9 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock(command): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - command, borg_local_path='borg', extra_environment=None, + command, + borg_local_path='borg', + extra_environment=None, ).once() @@ -33,7 +35,12 @@ def test_mount_archive_calls_borg_with_required_flags(): def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_archives_flags(): flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) insert_execute_command_mock( ('borg', 'mount', '--repo', 'repo', '--match-archives', 'archive', '/mnt') ) diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index e232df22..4da04dff 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -36,7 +36,12 @@ def test_create_repository_calls_borg_with_flags(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -51,7 +56,12 @@ def test_create_repository_with_dry_run_skips_borg_call(): insert_rinfo_command_not_found_mock() flexmock(module).should_receive('execute_command').never() flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=True, @@ -65,7 +75,12 @@ def test_create_repository_with_dry_run_skips_borg_call(): def test_create_repository_raises_for_borg_rcreate_error(): insert_rinfo_command_not_found_mock() flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').and_raise( module.subprocess.CalledProcessError(2, 'borg rcreate') @@ -84,7 +99,12 @@ def test_create_repository_raises_for_borg_rcreate_error(): def test_create_repository_skips_creation_when_repository_already_exists(): insert_rinfo_command_found_mock() flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -114,7 +134,12 @@ def test_create_repository_with_source_repository_calls_borg_with_other_repo_fla insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--other-repo', 'other.borg', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -130,7 +155,12 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--copy-crypt-key', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -146,7 +176,12 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--append-only', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -162,7 +197,12 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--storage-quota', '5G', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -178,7 +218,12 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--make-parent-dirs', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -195,7 +240,12 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): insert_rcreate_command_mock(RCREATE_COMMAND + ('--info', '--repo', 'repo')) insert_logging_mock(logging.INFO) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -211,7 +261,12 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): insert_rcreate_command_mock(RCREATE_COMMAND + ('--debug', '--repo', 'repo')) insert_logging_mock(logging.DEBUG) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -226,7 +281,12 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(('borg1',) + RCREATE_COMMAND[1:] + ('--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -242,7 +302,12 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--remote-path', 'borg1', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -258,7 +323,12 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--extra', '--options', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index ec0819cf..979b253e 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -11,7 +11,12 @@ def test_display_repository_info_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--repo', 'repo'), @@ -53,7 +58,12 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--info', '--repo', 'repo'), @@ -74,10 +84,16 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'rinfo', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.INFO) @@ -95,7 +111,12 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter( flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--debug', '--show-rc', '--repo', 'repo'), @@ -117,10 +138,16 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'rinfo', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.DEBUG) @@ -138,10 +165,16 @@ def test_display_repository_info_with_json_calls_borg_with_json_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'rinfo', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') json_output = module.display_repository_info( @@ -158,7 +191,12 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg1', 'rinfo', '--repo', 'repo'), @@ -180,7 +218,12 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_pa flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--remote-path', 'borg1', '--repo', 'repo'), @@ -203,7 +246,12 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parame flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--lock-wait', '5', '--repo', 'repo'), diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 6b1561a2..b83ba615 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -29,7 +29,8 @@ def test_resolve_archive_name_calls_borg_with_parameters(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') assert ( @@ -42,7 +43,8 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') insert_logging_mock(logging.INFO) @@ -56,7 +58,8 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter( expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') insert_logging_mock(logging.DEBUG) @@ -70,7 +73,8 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') assert ( @@ -100,7 +104,8 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param def test_resolve_archive_name_without_archives_raises(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return('') with pytest.raises(ValueError): @@ -374,7 +379,15 @@ def test_make_rlist_command_includes_short(): @pytest.mark.parametrize( 'argument_name', - ('sort_by', 'first', 'last', 'exclude', 'exclude_from', 'pattern', 'patterns_from',), + ( + 'sort_by', + 'first', + 'last', + 'exclude', + 'exclude_from', + 'pattern', + 'patterns_from', + ), ) def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) @@ -411,7 +424,9 @@ def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_p None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'foo-*', None, '1.2.3', + 'foo-*', + None, + '1.2.3', ).and_return(('--match-archives', 'foo-*')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) diff --git a/tests/unit/borg/test_version.py b/tests/unit/borg/test_version.py index 66789a8c..a051f693 100644 --- a/tests/unit/borg/test_version.py +++ b/tests/unit/borg/test_version.py @@ -15,7 +15,8 @@ def insert_execute_command_and_capture_output_mock( ): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - command, extra_environment=None, + command, + extra_environment=None, ).once().and_return(version_output) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 56f2332c..bd98c01f 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -229,7 +229,8 @@ def test_run_configuration_retries_hard_error(): ).and_return([flexmock()]) error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', OSError, + 'foo: Error running actions for repository', + OSError, ).and_return(error_logs) config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 1d61a771..63e3187a 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -21,13 +21,21 @@ from borgmatic.config import normalize as module {'location': {'source_directories': ['foo', 'bar']}}, False, ), - ({'location': None}, {'location': None}, False,), + ( + {'location': None}, + {'location': None}, + False, + ), ( {'storage': {'compression': 'yes_please'}}, {'storage': {'compression': 'yes_please'}}, False, ), - ({'storage': None}, {'storage': None}, False,), + ( + {'storage': None}, + {'storage': None}, + False, + ), ( {'hooks': {'healthchecks': 'https://example.com'}}, {'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}}, @@ -48,10 +56,9 @@ from borgmatic.config import normalize as module {'hooks': {'cronhub': {'ping_url': 'https://example.com'}}}, False, ), - ({'hooks': None}, {'hooks': None}, False,), ( - {'consistency': {'checks': ['archives']}}, - {'consistency': {'checks': [{'name': 'archives'}]}}, + {'hooks': None}, + {'hooks': None}, False, ), ( @@ -59,9 +66,26 @@ from borgmatic.config import normalize as module {'consistency': {'checks': [{'name': 'archives'}]}}, False, ), - ({'consistency': None}, {'consistency': None}, False,), - ({'location': {'numeric_owner': False}}, {'location': {'numeric_ids': False}}, False,), - ({'location': {'bsd_flags': False}}, {'location': {'flags': False}}, False,), + ( + {'consistency': {'checks': ['archives']}}, + {'consistency': {'checks': [{'name': 'archives'}]}}, + False, + ), + ( + {'consistency': None}, + {'consistency': None}, + False, + ), + ( + {'location': {'numeric_owner': False}}, + {'location': {'numeric_ids': False}}, + False, + ), + ( + {'location': {'bsd_flags': False}}, + {'location': {'flags': False}}, + False, + ), ( {'storage': {'remote_rate_limit': False}}, {'storage': {'upload_rate_limit': False}}, diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 11f3127c..24105547 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -138,7 +138,6 @@ def test_guard_configuration_contains_repository_does_not_raise_when_repository_ def test_guard_configuration_contains_repository_does_not_raise_when_repository_label_in_config(): - module.guard_configuration_contains_repository( repository='repo', configurations={ @@ -190,13 +189,15 @@ def test_guard_single_repository_selected_raises_when_multiple_repositories_conf def test_guard_single_repository_selected_does_not_raise_when_single_repository_configured_and_none_selected(): module.guard_single_repository_selected( - repository=None, configurations={'config.yaml': {'location': {'repositories': ['repo']}}}, + repository=None, + configurations={'config.yaml': {'location': {'repositories': ['repo']}}}, ) def test_guard_single_repository_selected_does_not_raise_when_no_repositories_configured_and_one_selected(): module.guard_single_repository_selected( - repository='repo', configurations={'config.yaml': {'location': {'repositories': []}}}, + repository='repo', + configurations={'config.yaml': {'location': {'repositories': []}}}, ) diff --git a/tests/unit/hooks/test_cronhub.py b/tests/unit/hooks/test_cronhub.py index f470b88e..2941592b 100644 --- a/tests/unit/hooks/test_cronhub.py +++ b/tests/unit/hooks/test_cronhub.py @@ -108,5 +108,9 @@ def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, ) diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py index 7ec1e2e6..12b96855 100644 --- a/tests/unit/hooks/test_cronitor.py +++ b/tests/unit/hooks/test_cronitor.py @@ -93,5 +93,9 @@ def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, ) diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index c975e4fd..5c6977da 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -206,7 +206,9 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): payload = 'data' flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload) flexmock(module.requests).should_receive('post').with_args( - f"https://hc-ping.com/{hook_config['ping_url']}", data=payload.encode('utf-8'), verify=True, + f"https://hc-ping.com/{hook_config['ping_url']}", + data=payload.encode('utf-8'), + verify=True, ).and_return(flexmock(ok=True)) module.ping_monitor( diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 77b830b7..f038a881 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -114,7 +114,8 @@ def test_dump_databases_runs_mongodump_with_directory_format(): flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').with_args( - ['mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'], shell=True, + ['mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'], + shell=True, ).and_return(flexmock()).once() assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [] diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 137a88a5..da5da16d 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -149,7 +149,14 @@ def test_execute_dump_command_runs_mysqldump(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--add-drop-database', '--databases', 'foo', '--result-file', 'dump',), + ( + 'mysqldump', + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -175,7 +182,13 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--databases', 'foo', '--result-file', 'dump',), + ( + 'mysqldump', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment=None, run_to_completion=False, ).and_return(process).once() diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 14924078..a6dd9d86 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -320,7 +320,11 @@ def test_execute_command_and_capture_output_returns_output_with_extra_environmen expected_output = '[]' flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( - full_command, stderr=None, shell=False, env={'a': 'b', 'c': 'd'}, cwd=None, + full_command, + stderr=None, + shell=False, + env={'a': 'b', 'c': 'd'}, + cwd=None, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( From 8cb5a42a9e156f126faf3797fff55a8bf95f4b61 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 21:21:25 -0700 Subject: [PATCH 91/94] Drop deprecated pkg_resources in favor of importlib.metadata and packaging. --- borgmatic/borg/feature.py | 26 +++++++++++++------------- borgmatic/commands/borgmatic.py | 8 ++++++-- borgmatic/config/validate.py | 17 +++++++++++++++-- setup.cfg | 2 -- setup.py | 1 + test_requirements.txt | 4 ++++ tests/unit/config/test_validate.py | 22 ++++++++++++++++++++++ 7 files changed, 61 insertions(+), 19 deletions(-) diff --git a/borgmatic/borg/feature.py b/borgmatic/borg/feature.py index 5294121d..b9311cd1 100644 --- a/borgmatic/borg/feature.py +++ b/borgmatic/borg/feature.py @@ -1,6 +1,6 @@ from enum import Enum -from pkg_resources import parse_version +from packaging.version import parse class Feature(Enum): @@ -18,17 +18,17 @@ class Feature(Enum): FEATURE_TO_MINIMUM_BORG_VERSION = { - Feature.COMPACT: parse_version('1.2.0a2'), # borg compact - Feature.ATIME: parse_version('1.2.0a7'), # borg create --atime - Feature.NOFLAGS: parse_version('1.2.0a8'), # borg create --noflags - Feature.NUMERIC_IDS: parse_version('1.2.0b3'), # borg create/extract/mount --numeric-ids - Feature.UPLOAD_RATELIMIT: parse_version('1.2.0b3'), # borg create --upload-ratelimit - Feature.SEPARATE_REPOSITORY_ARCHIVE: parse_version('2.0.0a2'), # --repo with separate archive - Feature.RCREATE: parse_version('2.0.0a2'), # borg rcreate - Feature.RLIST: parse_version('2.0.0a2'), # borg rlist - Feature.RINFO: parse_version('2.0.0a2'), # borg rinfo - Feature.MATCH_ARCHIVES: parse_version('2.0.0b3'), # borg --match-archives - Feature.EXCLUDED_FILES_MINUS: parse_version('2.0.0b5'), # --list --filter uses "-" for excludes + Feature.COMPACT: parse('1.2.0a2'), # borg compact + Feature.ATIME: parse('1.2.0a7'), # borg create --atime + Feature.NOFLAGS: parse('1.2.0a8'), # borg create --noflags + Feature.NUMERIC_IDS: parse('1.2.0b3'), # borg create/extract/mount --numeric-ids + Feature.UPLOAD_RATELIMIT: parse('1.2.0b3'), # borg create --upload-ratelimit + Feature.SEPARATE_REPOSITORY_ARCHIVE: parse('2.0.0a2'), # --repo with separate archive + Feature.RCREATE: parse('2.0.0a2'), # borg rcreate + Feature.RLIST: parse('2.0.0a2'), # borg rlist + Feature.RINFO: parse('2.0.0a2'), # borg rinfo + Feature.MATCH_ARCHIVES: parse('2.0.0b3'), # borg --match-archives + Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'), # --list --filter uses "-" for excludes } @@ -37,4 +37,4 @@ def available(feature, borg_version): Given a Borg Feature constant and a Borg version string, return whether that feature is available in that version of Borg. ''' - return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse_version(borg_version) + return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse(borg_version) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fece475a..999e9d8e 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -8,7 +8,11 @@ from queue import Queue from subprocess import CalledProcessError import colorama -import pkg_resources + +try: + import importlib_metadata +except ModuleNotFoundError: # pragma: nocover + import importlib.metadata as importlib_metadata import borgmatic.actions.borg import borgmatic.actions.break_lock @@ -706,7 +710,7 @@ def main(): # pragma: no cover global_arguments = arguments['global'] if global_arguments.version: - print(pkg_resources.require('borgmatic')[0].version) + print(importlib_metadata.version('borgmatic')) sys.exit(0) if global_arguments.bash_completion: print(borgmatic.commands.completion.bash_completion()) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index fcf29d38..537f4bee 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -1,9 +1,13 @@ import os import jsonschema -import pkg_resources import ruamel.yaml +try: + import importlib_metadata +except ModuleNotFoundError: # pragma: nocover + import importlib.metadata as importlib_metadata + from borgmatic.config import environment, load, normalize, override @@ -11,8 +15,17 @@ def schema_filename(): ''' Path to the installed YAML configuration schema file, used to validate and parse the configuration. + + Raise FileNotFoundError when the schema path does not exist. ''' - return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml') + try: + return next( + str(path.locate()) + for path in importlib_metadata.files('borgmatic') + if path.match('config/schema.yaml') + ) + except StopIteration: + raise FileNotFoundError('Configuration file schema could not be found') def format_json_error_path_element(path_element): diff --git a/setup.cfg b/setup.cfg index 30841386..a5ba3a6a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,8 +4,6 @@ description_file=README.md [tool:pytest] testpaths = tests addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end -filterwarnings = - ignore:Deprecated call to `pkg_resources.declare_namespace\('ruamel'\)`.*:DeprecationWarning [flake8] max-line-length = 100 diff --git a/setup.py b/setup.py index f6fca3db..ce6b78e5 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ setup( install_requires=( 'colorama>=0.4.1,<0.5', 'jsonschema', + 'packaging', 'requests', 'ruamel.yaml>0.15.0,<0.18.0', 'setuptools', diff --git a/test_requirements.txt b/test_requirements.txt index 67e02563..6516a500 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -12,8 +12,10 @@ flake8-use-fstring==1.4 flake8-variables-names==0.0.5 flexmock==0.11.3 idna==3.4 +importlib_metadata==6.3.0; python_version < '3.8' isort==5.12.0 mccabe==0.7.0 +packaging==23.1 pluggy==1.0.0 pathspec==0.11.1; python_version >= '3.8' py==1.11.0 @@ -27,3 +29,5 @@ requests==2.28.2 ruamel.yaml>0.15.0,<0.18.0 toml==0.10.2; python_version >= '3.8' typed-ast; python_version >= '3.8' +typing-extensions==4.5.0; python_version < '3.8' +zipp==3.15.0; python_version < '3.8' diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 24105547..e2b9f98f 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -4,6 +4,28 @@ from flexmock import flexmock from borgmatic.config import validate as module +def test_schema_filename_finds_schema_path(): + schema_path = '/var/borgmatic/config/schema.yaml' + + flexmock(module.importlib_metadata).should_receive('files').and_return( + flexmock(match=lambda path: False, locate=lambda: None), + flexmock(match=lambda path: True, locate=lambda: schema_path), + flexmock(match=lambda path: False, locate=lambda: None), + ) + + assert module.schema_filename() == schema_path + + +def test_schema_filename_with_missing_schema_path_raises(): + flexmock(module.importlib_metadata).should_receive('files').and_return( + flexmock(match=lambda path: False, locate=lambda: None), + flexmock(match=lambda path: False, locate=lambda: None), + ) + + with pytest.raises(FileNotFoundError): + assert module.schema_filename() + + def test_format_json_error_path_element_formats_array_index(): module.format_json_error_path_element(3) == '[3]' From 7ee37a890ed66d8529b93f5fb27d79ac3ee5a0d8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 23:22:07 -0700 Subject: [PATCH 92/94] Fix broken end-to-end tests by no longer using an editable package there, a work-around for https://github.com/pypa/packaging-problems/issues/609 --- tests/end-to-end/docker-compose.yaml | 1 + tox.ini | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index e6b0bf2b..0bbec8cc 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -23,6 +23,7 @@ services: - "../..:/app:ro" tmpfs: - "/app/borgmatic.egg-info" + - "/app/build" tty: true working_dir: /app entrypoint: /app/scripts/run-full-tests diff --git a/tox.ini b/tox.ini index b81588af..3a2b4764 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,9 @@ commands = pytest {posargs} [testenv:end-to-end] +usedevelop = False deps = -rtest_requirements.txt + . passenv = COVERAGE_FILE commands = pytest {posargs} --no-cov tests/end-to-end From 28e62d824b0f1a23dbb4e861b9576f61e0f2aa8a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 23:28:07 -0700 Subject: [PATCH 93/94] Upgrade end-to-end test packages. --- scripts/run-full-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run-full-tests b/scripts/run-full-tests index bf26c212..fb611fc4 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -21,8 +21,8 @@ apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client m py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true -python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 -pip3 install --ignore-installed tox==3.25.1 +python3 -m pip install --no-cache --upgrade pip==23.0.1 setuptools==67.6.1 +pip3 install --ignore-installed tox==4.4.12 export COVERAGE_FILE=/tmp/.coverage if [ "$1" != "--end-to-end-only" ] ; then From 25506b8d2c7fde14427957ec33d93de10dd3d999 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 23:47:51 -0700 Subject: [PATCH 94/94] Backing out upgrade of end-to-end test packages, because apparently we can't have nice things. --- scripts/run-full-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run-full-tests b/scripts/run-full-tests index fb611fc4..bf26c212 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -21,8 +21,8 @@ apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client m py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true -python3 -m pip install --no-cache --upgrade pip==23.0.1 setuptools==67.6.1 -pip3 install --ignore-installed tox==4.4.12 +python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 +pip3 install --ignore-installed tox==3.25.1 export COVERAGE_FILE=/tmp/.coverage if [ "$1" != "--end-to-end-only" ] ; then