From 8a63c494984425f649375475b07215c9614598f6 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 23 Mar 2023 01:01:26 +0530 Subject: [PATCH 001/344] feat: tag repos --- borgmatic/actions/create.py | 8 +++---- borgmatic/commands/borgmatic.py | 18 ++++++++-------- borgmatic/config/normalize.py | 30 +++++++++++++++++++++----- borgmatic/config/schema.yaml | 37 +++++++++++++++++++++------------ borgmatic/config/validate.py | 11 ++++++---- 5 files changed, 69 insertions(+), 35 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 96a48521..1ef0ecab 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -42,11 +42,11 @@ def run_create( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) + logger.info('{}: Creating archive{}'.format(repository['path'], dry_run_label)) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, @@ -54,7 +54,7 @@ def run_create( active_dumps = borgmatic.hooks.dispatch.call_hooks( 'dump_databases', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, @@ -63,7 +63,7 @@ def run_create( json_output = borgmatic.borg.create.create_archive( global_arguments.dry_run, - repository, + repository['path'], location, storage, local_borg_version, diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fe07981e..1f8b15f4 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -108,7 +108,7 @@ def run_configuration(config_filename, config, arguments): repo_queue.put((repo, 0),) while not repo_queue.empty(): - repository_path, retry_num = repo_queue.get() + repository, retry_num = repo_queue.get() timeout = retry_num * retry_wait if timeout: logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') @@ -125,14 +125,14 @@ def run_configuration(config_filename, config, arguments): local_path=local_path, remote_path=remote_path, local_borg_version=local_borg_version, - repository_path=repository_path, + repository=repository, ) except (OSError, CalledProcessError, ValueError) as error: if retry_num < retries: - repo_queue.put((repository_path, retry_num + 1),) + repo_queue.put((repository, retry_num + 1),) tuple( # Consume the generator so as to trigger logging. log_error_records( - '{}: Error running actions for repository'.format(repository_path), + '{}: Error running actions for repository'.format(repository['path']), error, levelno=logging.WARNING, log_command_error_output=True, @@ -147,10 +147,10 @@ def run_configuration(config_filename, config, arguments): return yield from log_error_records( - '{}: Error running actions for repository'.format(repository_path), error + '{}: Error running actions for repository'.format(repository['path']), error ) encountered_error = error - error_repository = repository_path + error_repository = repository['path'] try: if using_primary_action: @@ -248,7 +248,7 @@ def run_actions( local_path, remote_path, local_borg_version, - repository_path, + repository, ): ''' Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration @@ -263,13 +263,13 @@ def run_actions( invalid. ''' add_custom_log_levels() - repository = os.path.expanduser(repository_path) + repository_path = os.path.expanduser(repository['path']) global_arguments = arguments['global'] dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' hook_context = { 'repository': repository_path, # Deprecated: For backwards compatibility with borgmatic < 1.6.0. - 'repositories': ','.join(location['repositories']), + 'repositories': ','.join([repo['path'] for repo in location['repositories']]), } command.execute_hook( diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index a143a192..58e34680 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -56,9 +56,13 @@ def normalize(config_filename, config): # Upgrade remote repositories to ssh:// syntax, required in Borg 2. repositories = location.get('repositories') + if isinstance(repositories[0], str): + config['location']['repositories'] = [{'path': repository} for repository in repositories] + repositories = config['location']['repositories'] if repositories: config['location']['repositories'] = [] - for repository in repositories: + for repository_dict in repositories: + repository = repository_dict['path'] if '~' in repository: logs.append( logging.makeLogRecord( @@ -71,11 +75,19 @@ def normalize(config_filename, config): ) if ':' in repository: if repository.startswith('file://'): + updated_repository_path = os.path.abspath(repository.partition('file://')[-1]) + config['location']['repositories'].append( - os.path.abspath(repository.partition('file://')[-1]) + { + 'path': updated_repository_path, + 'label': repository_dict.get('label', None), + } ) elif repository.startswith('ssh://'): - config['location']['repositories'].append(repository) + config['location']['repositories'].append({ + 'path': repository, + 'label': repository_dict.get('label', None), + }) else: rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( @@ -87,8 +99,16 @@ def normalize(config_filename, config): ) ) ) - config['location']['repositories'].append(rewritten_repository) + config['location']['repositories'].append({ + 'path': rewritten_repository, + 'label': repository_dict.get('label', None), + }) else: - config['location']['repositories'].append(repository) + config['location']['repositories'].append( + { + 'path': repository, + 'label': repository_dict.get('label', None), + } + ) return logs diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d4d57ab6..4235adba 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -29,19 +29,30 @@ properties: repositories: type: array items: - type: string - description: | - Paths to local or remote repositories (required). Tildes are - expanded. Multiple repositories are backed up to in - sequence. Borg placeholders can be used. See the output of - "borg help placeholders" for details. See ssh_command for - SSH options like identity file or port. If systemd service - is used, then add local repository paths in the systemd - service file to the ReadWritePaths list. - example: - - ssh://user@backupserver/./sourcehostname.borg - - ssh://user@backupserver/./{fqdn} - - /var/local/backups/local.borg + type: object + required: + - path + properties: + path: + type: string + description: | + Path to local or remote repository (required). + are expanded. Multiple repositories are backed up to + in sequence. Borg placeholders can be used. See the + output of "borg help placeholders" for details. See + ssh_command for SSH options like identity file or + port. If systemd service is used, then add local + repository paths in the systemd service file to the + ReadWritePaths list. + example: + - ssh://user@backupserver/./sourcehostname.borg + - ssh://user@backupserver/./{fqdn} + - /var/local/backups/local.borg + label: + type: string + description: | + Optional label for the repository. + example: backupserver working_directory: type: string description: | diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 5828380e..9b64377a 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -138,10 +138,13 @@ def normalize_repository_path(repository): def repositories_match(first, second): ''' - Given two repository paths (relative and/or absolute), return whether they match. + Given two repository dicts with keys 'path' (relative and/or absolute), + and 'label', return whether they match. ''' - return normalize_repository_path(first) == normalize_repository_path(second) - + if isinstance(first,str) and isinstance(second,str): + return normalize_repository_path(first) == normalize_repository_path(second) + elif isinstance(first,dict) and isinstance(second,str): + return (second == first.get('label')) or (normalize_repository_path(second) == normalize_repository_path(first.get('path'))) def guard_configuration_contains_repository(repository, configurations): ''' @@ -160,7 +163,7 @@ def guard_configuration_contains_repository(repository, configurations): config_repository for config in configurations.values() for config_repository in config['location']['repositories'] - if repositories_match(repository, config_repository) + if repositories_match(config_repository, repository) ) ) From 7a2f287918beea36378aa0dd9fbdf8ca9ae16355 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 23 Mar 2023 01:08:30 +0530 Subject: [PATCH 002/344] reformat base --- borgmatic/config/normalize.py | 19 +++++++------------ borgmatic/config/validate.py | 9 ++++++--- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 58e34680..8ce4af22 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -84,10 +84,9 @@ def normalize(config_filename, config): } ) elif repository.startswith('ssh://'): - config['location']['repositories'].append({ - 'path': repository, - 'label': repository_dict.get('label', None), - }) + config['location']['repositories'].append( + {'path': repository, 'label': repository_dict.get('label', None),} + ) else: rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( @@ -99,16 +98,12 @@ def normalize(config_filename, config): ) ) ) - config['location']['repositories'].append({ - 'path': rewritten_repository, - 'label': repository_dict.get('label', None), - }) + config['location']['repositories'].append( + {'path': rewritten_repository, 'label': repository_dict.get('label', None),} + ) else: config['location']['repositories'].append( - { - 'path': repository, - 'label': repository_dict.get('label', None), - } + {'path': repository, 'label': repository_dict.get('label', None),} ) return logs diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 9b64377a..b43d42df 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -141,10 +141,13 @@ def repositories_match(first, second): Given two repository dicts with keys 'path' (relative and/or absolute), and 'label', return whether they match. ''' - if isinstance(first,str) and isinstance(second,str): + if isinstance(first, str) and isinstance(second, str): return normalize_repository_path(first) == normalize_repository_path(second) - elif isinstance(first,dict) and isinstance(second,str): - return (second == first.get('label')) or (normalize_repository_path(second) == normalize_repository_path(first.get('path'))) + elif isinstance(first, dict) and isinstance(second, str): + return (second == first.get('label')) or ( + normalize_repository_path(second) == normalize_repository_path(first.get('path')) + ) + def guard_configuration_contains_repository(repository, configurations): ''' From e83ad9e1e4cb3dfb2209268f51a0f853825c6fc4 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 01:04:57 +0530 Subject: [PATCH 003/344] use repository["path"] instead of repository --- borgmatic/actions/borg.py | 6 ++-- borgmatic/actions/break_lock.py | 8 +++-- borgmatic/actions/check.py | 4 +-- borgmatic/actions/compact.py | 8 +++-- borgmatic/actions/export_tar.py | 8 +++-- borgmatic/actions/extract.py | 8 +++-- borgmatic/actions/info.py | 6 ++-- borgmatic/actions/list.py | 8 ++--- borgmatic/actions/mount.py | 10 +++--- borgmatic/actions/prune.py | 4 +-- borgmatic/actions/rcreate.py | 4 +-- borgmatic/actions/restore.py | 27 ++++++++++++----- borgmatic/actions/rinfo.py | 6 ++-- borgmatic/actions/rlist.py | 4 +-- borgmatic/commands/borgmatic.py | 4 +++ borgmatic/config/normalize.py | 37 ++++++++++++++--------- borgmatic/config/schema.yaml | 36 ++++++++++++++-------- borgmatic/config/validate.py | 14 +++++---- tests/integration/config/test_validate.py | 21 +++++++++---- tests/unit/actions/test_borg.py | 2 +- tests/unit/actions/test_break_lock.py | 2 +- tests/unit/actions/test_check.py | 6 ++-- tests/unit/actions/test_compact.py | 6 ++-- tests/unit/actions/test_create.py | 4 +-- tests/unit/actions/test_export_tar.py | 2 +- tests/unit/actions/test_extract.py | 2 +- tests/unit/actions/test_info.py | 2 +- tests/unit/actions/test_list.py | 2 +- tests/unit/actions/test_mount.py | 2 +- tests/unit/actions/test_prune.py | 4 +-- tests/unit/actions/test_rcreate.py | 4 +-- tests/unit/actions/test_restore.py | 16 +++++----- tests/unit/actions/test_rinfo.py | 2 +- tests/unit/actions/test_rlist.py | 2 +- tests/unit/commands/test_borgmatic.py | 36 +++++++++++----------- tests/unit/config/test_normalize.py | 10 +++--- 36 files changed, 193 insertions(+), 134 deletions(-) diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index a50dd286..bf9abd5a 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -16,9 +16,9 @@ def run_borg( if borg_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, borg_arguments.repository ): - logger.info('{}: Running arbitrary Borg command'.format(repository)) + logger.info('{}: Running arbitrary Borg command'.format(repository['path'])) archive_name = borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], borg_arguments.archive, storage, local_borg_version, @@ -26,7 +26,7 @@ def run_borg( remote_path, ) borgmatic.borg.borg.run_arbitrary_borg( - repository, + repository['path'], storage, local_borg_version, options=borg_arguments.options, diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index 65384d7a..eb6e4547 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -15,7 +15,11 @@ def run_break_lock( if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, break_lock_arguments.repository ): - logger.info(f'{repository}: Breaking repository and cache locks') + logger.info(f'{repository["path"]}: Breaking repository and cache locks') borgmatic.borg.break_lock.break_lock( - repository, storage, local_borg_version, local_path=local_path, remote_path=remote_path, + repository['path'], + storage, + local_borg_version, + local_path=local_path, + remote_path=remote_path, ) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index f3572395..ff8225ec 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -37,9 +37,9 @@ def run_check( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Running consistency checks'.format(repository)) + logger.info('{}: Running consistency checks'.format(repository['path'])) borgmatic.borg.check.check_archives( - repository, + repository['path'], location, storage, consistency, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 7a25b829..13bb28ea 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -39,10 +39,10 @@ def run_compact( **hook_context, ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): - logger.info('{}: Compacting segments{}'.format(repository, dry_run_label)) + logger.info('{}: Compacting segments{}'.format(repository['path'], dry_run_label)) borgmatic.borg.compact.compact_segments( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, local_path=local_path, @@ -52,7 +52,9 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover - logger.info('{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository)) + logger.info( + '{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository['path']) + ) borgmatic.hooks.command.execute_hook( hooks.get('after_compact'), hooks.get('umask'), diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index ae349208..7599a23a 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -23,13 +23,15 @@ def run_export_tar( repository, export_tar_arguments.repository ): logger.info( - '{}: Exporting archive {} as tar file'.format(repository, export_tar_arguments.archive) + '{}: Exporting archive {} as tar file'.format( + repository['path'], export_tar_arguments.archive + ) ) borgmatic.borg.export_tar.export_tar_archive( global_arguments.dry_run, - repository, + repository['path'], borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], export_tar_arguments.archive, storage, local_borg_version, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index a3d89a55..a36c498b 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -35,12 +35,14 @@ def run_extract( if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): - logger.info('{}: Extracting archive {}'.format(repository, extract_arguments.archive)) + logger.info( + '{}: Extracting archive {}'.format(repository['path'], extract_arguments.archive) + ) borgmatic.borg.extract.extract_archive( global_arguments.dry_run, - repository, + repository['path'], borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], extract_arguments.archive, storage, local_borg_version, diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index ab4fe426..6cd775f5 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -20,9 +20,9 @@ def run_info( repository, info_arguments.repository ): if not info_arguments.json: # pragma: nocover - logger.answer(f'{repository}: Displaying archive summary information') + logger.answer(f'{repository["path"]}: Displaying archive summary information') info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], info_arguments.archive, storage, local_borg_version, @@ -30,7 +30,7 @@ def run_info( remote_path, ) json_output = borgmatic.borg.info.display_archives_info( - repository, + repository['path'], storage, local_borg_version, info_arguments=info_arguments, diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 78efdf59..0e736f84 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -20,11 +20,11 @@ def run_list( ): if not list_arguments.json: # pragma: nocover if list_arguments.find_paths: - logger.answer(f'{repository}: Searching archives') + logger.answer(f'{repository["path"]}: Searching archives') elif not list_arguments.archive: - logger.answer(f'{repository}: Listing archives') + logger.answer(f'{repository["path"]}: Listing archives') list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], list_arguments.archive, storage, local_borg_version, @@ -32,7 +32,7 @@ def run_list( remote_path, ) json_output = borgmatic.borg.list.list_archive( - repository, + repository['path'], storage, local_borg_version, list_arguments=list_arguments, diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index e2703a57..d2c1821d 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -17,14 +17,16 @@ def run_mount( repository, mount_arguments.repository ): if mount_arguments.archive: - logger.info('{}: Mounting archive {}'.format(repository, mount_arguments.archive)) + logger.info( + '{}: Mounting archive {}'.format(repository['path'], mount_arguments.archive) + ) else: # pragma: nocover - logger.info('{}: Mounting repository'.format(repository)) + logger.info('{}: Mounting repository'.format(repository['path'])) borgmatic.borg.mount.mount_archive( - repository, + repository['path'], borgmatic.borg.rlist.resolve_archive_name( - repository, + repository['path'], mount_arguments.archive, storage, local_borg_version, diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index ca098ce4..6ba28a92 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -37,10 +37,10 @@ def run_prune( global_arguments.dry_run, **hook_context, ) - logger.info('{}: Pruning archives{}'.format(repository, dry_run_label)) + logger.info('{}: Pruning archives{}'.format(repository['path'], dry_run_label)) borgmatic.borg.prune.prune_archives( global_arguments.dry_run, - repository, + repository['path'], storage, retention, local_borg_version, diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index 0052b4b6..92dcfe34 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -23,10 +23,10 @@ def run_rcreate( ): return - logger.info('{}: Creating repository'.format(repository)) + logger.info('{}: Creating repository'.format(repository['path'])) borgmatic.borg.rcreate.create_repository( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, rcreate_arguments.encryption_mode, diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 7a058092..793b8698 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -256,22 +256,35 @@ def run_restore( return logger.info( - '{}: Restoring databases from archive {}'.format(repository, restore_arguments.archive) + '{}: Restoring databases from archive {}'.format( + repository['path'], restore_arguments.archive + ) ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, ) archive_name = borgmatic.borg.rlist.resolve_archive_name( - repository, restore_arguments.archive, storage, local_borg_version, local_path, remote_path, + repository['path'], + restore_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, ) archive_database_names = collect_archive_database_names( - repository, archive_name, location, storage, local_borg_version, local_path, remote_path, + repository['path'], + archive_name, + location, + storage, + local_borg_version, + local_path, + remote_path, ) restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names) found_names = set() @@ -291,7 +304,7 @@ def run_restore( found_names.add(database_name) restore_single_database( - repository, + repository['path'], location, storage, hooks, @@ -320,7 +333,7 @@ def run_restore( database['name'] = database_name restore_single_database( - repository, + repository['path'], location, storage, hooks, @@ -336,7 +349,7 @@ def run_restore( borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 611d1bc2..5e0a41a7 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -19,9 +19,11 @@ def run_rinfo( repository, rinfo_arguments.repository ): if not rinfo_arguments.json: # pragma: nocover - logger.answer('{}: Displaying repository summary information'.format(repository)) + logger.answer( + '{}: Displaying repository summary information'.format(repository['path']) + ) json_output = borgmatic.borg.rinfo.display_repository_info( - repository, + repository['path'], storage, local_borg_version, rinfo_arguments=rinfo_arguments, diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index 72d52068..29e500c1 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -19,9 +19,9 @@ def run_rlist( repository, rlist_arguments.repository ): if not rlist_arguments.json: # pragma: nocover - logger.answer('{}: Listing repository'.format(repository)) + logger.answer('{}: Listing repository'.format(repository['path'])) json_output = borgmatic.borg.rlist.list_repository( - repository, + repository['path'], storage, local_borg_version, rlist_arguments=rlist_arguments, diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 1f8b15f4..b52bc568 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -109,6 +109,8 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository, retry_num = repo_queue.get() + if isinstance(repository, str): + repository = {'path': repository} timeout = retry_num * retry_wait if timeout: logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') @@ -263,6 +265,8 @@ def run_actions( invalid. ''' add_custom_log_levels() + if isinstance(repository, str): + repository = {'path': repository} repository_path = os.path.expanduser(repository['path']) global_arguments = arguments['global'] dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 8ce4af22..eb2ed013 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -56,14 +56,16 @@ def normalize(config_filename, config): # Upgrade remote repositories to ssh:// syntax, required in Borg 2. repositories = location.get('repositories') - if isinstance(repositories[0], str): - config['location']['repositories'] = [{'path': repository} for repository in repositories] - repositories = config['location']['repositories'] if repositories: + if isinstance(repositories[0], str): + config['location']['repositories'] = [ + {'path': repository} for repository in repositories + ] + repositories = config['location']['repositories'] config['location']['repositories'] = [] for repository_dict in repositories: - repository = repository_dict['path'] - if '~' in repository: + repository_path = repository_dict['path'] + if '~' in repository_path: logs.append( logging.makeLogRecord( dict( @@ -73,37 +75,42 @@ def normalize(config_filename, config): ) ) ) - if ':' in repository: - if repository.startswith('file://'): - updated_repository_path = os.path.abspath(repository.partition('file://')[-1]) + if ':' in repository_path: + if repository_path.startswith('file://'): + updated_repository_path = os.path.abspath( + repository_path.partition('file://')[-1] + ) config['location']['repositories'].append( { 'path': updated_repository_path, - 'label': repository_dict.get('label', None), + 'label': repository_dict.get('label', ''), } ) - elif repository.startswith('ssh://'): + elif repository_path.startswith('ssh://'): config['location']['repositories'].append( - {'path': repository, 'label': repository_dict.get('label', None),} + {'path': repository_path, 'label': repository_dict.get('label', '')} ) else: - rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" + rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( logging.makeLogRecord( dict( levelno=logging.WARNING, levelname='WARNING', - msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository}" as "{rewritten_repository}"', + msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository_path}" as "{rewritten_repository_path}"', ) ) ) config['location']['repositories'].append( - {'path': rewritten_repository, 'label': repository_dict.get('label', None),} + { + 'path': rewritten_repository_path, + 'label': repository_dict.get('label', ''), + } ) else: config['location']['repositories'].append( - {'path': repository, 'label': repository_dict.get('label', None),} + {'path': repository_path, 'label': repository_dict.get('label', '')} ) return logs diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 4235adba..14d028b7 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -36,23 +36,33 @@ properties: path: type: string description: | - Path to local or remote repository (required). - are expanded. Multiple repositories are backed up to - in sequence. Borg placeholders can be used. See the - output of "borg help placeholders" for details. See - ssh_command for SSH options like identity file or - port. If systemd service is used, then add local - repository paths in the systemd service file to the - ReadWritePaths list. - example: - - ssh://user@backupserver/./sourcehostname.borg - - ssh://user@backupserver/./{fqdn} - - /var/local/backups/local.borg + Path to local or remote repository + (required). Tildes are expanded. + Multiple repositories are backed up + to in sequence. Borg placeholders + can be used. See the output of + "borg help placeholders" for + details. See ssh_command for SSH + options like identity file or port. + If systemd service is used, then + add local repository paths in the + systemd service file to the + ReadWritePaths list. + example: ssh://user@backupserver/./{fqdn} label: type: string description: | - Optional label for the repository. + Optional label for the repository. This + can be used with the --repository option + to select a repository to backup to. + If not specified, the repository path is + used as the label. example: backupserver + example: + - path: ssh://user@backupserver/./{fqdn} + label: backupserver + - path: /mnt/backup + label: local working_directory: type: string description: | diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index b43d42df..ecacec71 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -141,12 +141,14 @@ def repositories_match(first, second): Given two repository dicts with keys 'path' (relative and/or absolute), and 'label', return whether they match. ''' - if isinstance(first, str) and isinstance(second, str): - return normalize_repository_path(first) == normalize_repository_path(second) - elif isinstance(first, dict) and isinstance(second, str): - return (second == first.get('label')) or ( - normalize_repository_path(second) == normalize_repository_path(first.get('path')) - ) + if isinstance(first, str): + first = {'path': first, 'label': first} + if isinstance(second, str): + second = {'path': second, 'label': second} + return (first.get('label') == second.get('label')) or ( + normalize_repository_path(first.get('path')) + == normalize_repository_path(second.get('path')) + ) def guard_configuration_contains_repository(repository, configurations): diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 5d948ae2..31a85cd1 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -63,7 +63,10 @@ def test_parse_configuration_transforms_file_into_mapping(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']}, + 'location': { + 'source_directories': ['/home', '/etc'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], + }, 'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60}, 'consistency': {'checks': [{'name': 'repository'}, {'name': 'archives'}]}, } @@ -89,7 +92,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): assert config == { 'location': { 'source_directories': [f'/home/{string.punctuation}'], - 'repositories': ['test.borg'], + 'repositories': [{'path': 'test.borg', 'label': ''}], } } assert logs == [] @@ -151,7 +154,10 @@ def test_parse_configuration_inlines_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']}, + 'location': { + 'source_directories': ['/home'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], + }, 'retention': {'keep_daily': 7, 'keep_hourly': 24}, } assert logs == [] @@ -185,7 +191,10 @@ def test_parse_configuration_merges_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']}, + 'location': { + 'source_directories': ['/home'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], + }, 'retention': {'keep_daily': 1, 'keep_hourly': 24}, } assert logs == [] @@ -247,7 +256,7 @@ def test_parse_configuration_applies_overrides(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': ['hostname.borg'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], 'local_path': 'borg2', } } @@ -273,7 +282,7 @@ def test_parse_configuration_applies_normalization(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': ['hostname.borg'], + 'repositories': [{'path': 'hostname.borg', 'label': ''}], 'exclude_if_present': ['.nobackup'], } } diff --git a/tests/unit/actions/test_borg.py b/tests/unit/actions/test_borg.py index 7b22c7b3..f597acbf 100644 --- a/tests/unit/actions/test_borg.py +++ b/tests/unit/actions/test_borg.py @@ -13,7 +13,7 @@ def test_run_borg_does_not_raise(): borg_arguments = flexmock(repository=flexmock(), archive=flexmock(), options=flexmock()) module.run_borg( - repository='repo', + repository={'path': 'repos'}, storage={}, local_borg_version=None, borg_arguments=borg_arguments, diff --git a/tests/unit/actions/test_break_lock.py b/tests/unit/actions/test_break_lock.py index c7db00bc..6dc2470e 100644 --- a/tests/unit/actions/test_break_lock.py +++ b/tests/unit/actions/test_break_lock.py @@ -10,7 +10,7 @@ def test_run_break_lock_does_not_raise(): break_lock_arguments = flexmock(repository=flexmock()) module.run_break_lock( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, break_lock_arguments=break_lock_arguments, diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 3e1a9c2f..4c2027eb 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -18,7 +18,7 @@ def test_run_check_calls_hooks_for_configured_repository(): module.run_check( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, consistency={}, @@ -49,7 +49,7 @@ def test_run_check_runs_with_selected_repository(): module.run_check( config_filename='test.yaml', - repository=flexmock(), + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, consistency={}, @@ -80,7 +80,7 @@ def test_run_check_bails_if_repository_does_not_match(): module.run_check( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, consistency={}, diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 4dae903e..fbd4f905 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -16,7 +16,7 @@ def test_compact_actions_calls_hooks_for_configured_repository(): module.run_compact( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, @@ -44,7 +44,7 @@ def test_compact_runs_with_selected_repository(): module.run_compact( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, @@ -72,7 +72,7 @@ def test_compact_bails_if_repository_does_not_match(): module.run_compact( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 8a9d0b4e..2b724085 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -24,7 +24,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): list( module.run_create( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={}, storage={}, hooks={}, @@ -57,7 +57,7 @@ def test_run_create_runs_with_selected_repository(): list( module.run_create( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={}, storage={}, hooks={}, diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index 41b680af..6741d427 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -19,7 +19,7 @@ def test_run_export_tar_does_not_raise(): global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_export_tar( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, export_tar_arguments=export_tar_arguments, diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index 4222b8a8..32b93b4e 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -20,7 +20,7 @@ def test_run_extract_calls_hooks(): module.run_extract( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, location={'repositories': ['repo']}, storage={}, hooks={}, diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py index 8cde178d..a4f1d544 100644 --- a/tests/unit/actions/test_info.py +++ b/tests/unit/actions/test_info.py @@ -14,7 +14,7 @@ def test_run_info_does_not_raise(): list( module.run_info( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, info_arguments=info_arguments, diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py index f4a603de..bfdfd010 100644 --- a/tests/unit/actions/test_list.py +++ b/tests/unit/actions/test_list.py @@ -14,7 +14,7 @@ def test_run_list_does_not_raise(): list( module.run_list( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, list_arguments=list_arguments, diff --git a/tests/unit/actions/test_mount.py b/tests/unit/actions/test_mount.py index 0624a896..7eadfca1 100644 --- a/tests/unit/actions/test_mount.py +++ b/tests/unit/actions/test_mount.py @@ -17,7 +17,7 @@ def test_run_mount_does_not_raise(): ) module.run_mount( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, mount_arguments=mount_arguments, diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index db9c1247..7af7ea77 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -13,7 +13,7 @@ def test_run_prune_calls_hooks_for_configured_repository(): module.run_prune( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, @@ -38,7 +38,7 @@ def test_run_prune_runs_with_selected_repository(): module.run_prune( config_filename='test.yaml', - repository='repo', + repository={'path': 'repo'}, storage={}, retention={}, hooks={}, diff --git a/tests/unit/actions/test_rcreate.py b/tests/unit/actions/test_rcreate.py index 78d4af1f..b77fa757 100644 --- a/tests/unit/actions/test_rcreate.py +++ b/tests/unit/actions/test_rcreate.py @@ -18,7 +18,7 @@ def test_run_rcreate_does_not_raise(): ) module.run_rcreate( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rcreate_arguments=arguments, @@ -45,7 +45,7 @@ def test_run_rcreate_bails_if_repository_does_not_match(): ) module.run_rcreate( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rcreate_arguments=arguments, diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index eaad6bf4..8255c87d 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -67,7 +67,7 @@ def test_collect_archive_database_names_parses_archive_paths(): ) archive_database_names = module.collect_archive_database_names( - repository='repo', + repository={'path': 'repo'}, archive='archive', location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), @@ -92,7 +92,7 @@ def test_collect_archive_database_names_parses_directory_format_archive_paths(): ) archive_database_names = module.collect_archive_database_names( - repository='repo', + repository={'path': 'repo'}, archive='archive', location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), @@ -113,7 +113,7 @@ def test_collect_archive_database_names_skips_bad_archive_paths(): ) archive_database_names = module.collect_archive_database_names( - repository='repo', + repository={'path': 'repo'}, archive='archive', location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), @@ -251,7 +251,7 @@ def test_run_restore_restores_each_database(): flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), @@ -273,7 +273,7 @@ def test_run_restore_bails_for_non_matching_repository(): flexmock(module).should_receive('restore_single_database').never() module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), @@ -345,7 +345,7 @@ def test_run_restore_restores_database_configured_with_all_name(): flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), @@ -417,7 +417,7 @@ def test_run_restore_skips_missing_database(): flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), @@ -483,7 +483,7 @@ def test_run_restore_restores_databases_from_different_hooks(): flexmock(module).should_receive('ensure_databases_found') module.run_restore( - repository='repo', + repository={'path': 'repo'}, location=flexmock(), storage=flexmock(), hooks=flexmock(), diff --git a/tests/unit/actions/test_rinfo.py b/tests/unit/actions/test_rinfo.py index d789ef11..133e61ac 100644 --- a/tests/unit/actions/test_rinfo.py +++ b/tests/unit/actions/test_rinfo.py @@ -11,7 +11,7 @@ def test_run_rinfo_does_not_raise(): list( module.run_rinfo( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rinfo_arguments=rinfo_arguments, diff --git a/tests/unit/actions/test_rlist.py b/tests/unit/actions/test_rlist.py index 3da0f905..7f8b58aa 100644 --- a/tests/unit/actions/test_rlist.py +++ b/tests/unit/actions/test_rlist.py @@ -11,7 +11,7 @@ def test_run_rlist_does_not_raise(): list( module.run_rlist( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, rlist_arguments=rlist_arguments, diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 19ac00de..14e7443e 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -409,7 +409,7 @@ def test_run_actions_runs_rcreate(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -431,7 +431,7 @@ def test_run_actions_runs_transfer(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -454,7 +454,7 @@ def test_run_actions_runs_create(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -477,7 +477,7 @@ def test_run_actions_runs_prune(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -499,7 +499,7 @@ def test_run_actions_runs_compact(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -522,7 +522,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -545,7 +545,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -567,7 +567,7 @@ def test_run_actions_runs_extract(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -589,7 +589,7 @@ def test_run_actions_runs_export_tar(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -611,7 +611,7 @@ def test_run_actions_runs_mount(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -633,7 +633,7 @@ def test_run_actions_runs_restore(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -656,7 +656,7 @@ def test_run_actions_runs_rlist(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -680,7 +680,7 @@ def test_run_actions_runs_list(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -704,7 +704,7 @@ def test_run_actions_runs_rinfo(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -728,7 +728,7 @@ def test_run_actions_runs_info(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) assert result == (expected,) @@ -751,7 +751,7 @@ def test_run_actions_runs_break_lock(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -773,7 +773,7 @@ def test_run_actions_runs_borg(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) @@ -800,7 +800,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository_path='repo', + repository='repo', ) ) diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 821c3202..ff67aa12 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -69,27 +69,27 @@ from borgmatic.config import normalize as module ), ( {'location': {'repositories': ['foo@bar:/repo']}}, - {'location': {'repositories': ['ssh://foo@bar/repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': ''}]}}, True, ), ( {'location': {'repositories': ['foo@bar:repo']}}, - {'location': {'repositories': ['ssh://foo@bar/./repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/./repo', 'label': ''}]}}, True, ), ( {'location': {'repositories': ['foo@bar:~/repo']}}, - {'location': {'repositories': ['ssh://foo@bar/~/repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo', 'label': ''}]}}, True, ), ( {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, - {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo', 'label': ''}]}}, False, ), ( {'location': {'repositories': ['file:///repo']}}, - {'location': {'repositories': ['/repo']}}, + {'location': {'repositories': [{'path': '/repo', 'label': ''}]}}, False, ), ), From aeaf69f49e0081a2b3c835d86d5206a550357155 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 01:34:03 +0530 Subject: [PATCH 004/344] pass all tests --- borgmatic/config/schema.yaml | 2 +- tests/end-to-end/test_borgmatic.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 14d028b7..e24b0eca 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -59,7 +59,7 @@ properties: used as the label. example: backupserver example: - - path: ssh://user@backupserver/./{fqdn} + - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver - path: /mnt/backup label: local diff --git a/tests/end-to-end/test_borgmatic.py b/tests/end-to-end/test_borgmatic.py index c2d10291..86cd83c2 100644 --- a/tests/end-to-end/test_borgmatic.py +++ b/tests/end-to-end/test_borgmatic.py @@ -19,9 +19,8 @@ def generate_configuration(config_path, repository_path): open(config_path) .read() .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) - .replace('- ssh://user@backupserver/./{fqdn}', '') - .replace('- /var/local/backups/local.borg', '') - .replace('- /home/user/path with spaces', '') + .replace('- path: /mnt/backup', '') + .replace('label: local', '') .replace('- /home', '- {}'.format(config_path)) .replace('- /etc', '') .replace('- /var/log/syslog*', '') From d6dfb8753a0a09cc2998fabd93f8fad78efd4850 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 25 Mar 2023 01:50:47 +0530 Subject: [PATCH 005/344] reformat --- borgmatic/actions/compact.py | 8 +------- borgmatic/actions/export_tar.py | 4 +++- borgmatic/actions/restore.py | 4 +++- borgmatic/commands/borgmatic.py | 2 +- tests/end-to-end/test_borgmatic.py | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index e3018af5..95334c52 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -52,13 +52,7 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover -<<<<<<< HEAD - logger.info( - '{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository['path']) - ) -======= - logger.info(f'{repository}: Skipping compact (only available/needed in Borg 1.2+)') ->>>>>>> f42890430c59a40a17d9a68a193d6a09674770cb + logger.info(f'{repository["path"]}: Skipping compact (only available/needed in Borg 1.2+)') borgmatic.hooks.command.execute_hook( hooks.get('after_compact'), hooks.get('umask'), diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index 37a870a8..ff9f31ba 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -22,7 +22,9 @@ def run_export_tar( if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, export_tar_arguments.repository ): - logger.info(f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file') + logger.info( + f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file' + ) borgmatic.borg.export_tar.export_tar_archive( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index e718e74e..bbbb4c74 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -255,7 +255,9 @@ def run_restore( ): return - logger.info(f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}') + logger.info( + f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}' + ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 86f047ec..7612970c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -169,7 +169,7 @@ def run_configuration(config_filename, config, arguments): return encountered_error = error - yield from log_error_records(f'{repository_path}: Error pinging monitor', error) + yield from log_error_records(f'{repository["path"]}: Error pinging monitor', error) if not encountered_error: try: diff --git a/tests/end-to-end/test_borgmatic.py b/tests/end-to-end/test_borgmatic.py index c1653d2f..5e915f00 100644 --- a/tests/end-to-end/test_borgmatic.py +++ b/tests/end-to-end/test_borgmatic.py @@ -19,7 +19,7 @@ def generate_configuration(config_path, repository_path): .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) .replace('- path: /mnt/backup', '') .replace('label: local', '') - .replace('- /home', '- {}'.format(config_path)) + .replace('- /home', f'- {config_path}') .replace('- /etc', '') .replace('- /var/log/syslog*', '') + 'storage:\n encryption_passphrase: "test"' From 141474ff0761f76485b2ffd54d91168277617913 Mon Sep 17 00:00:00 2001 From: jetchirag Date: Sun, 26 Mar 2023 01:58:03 +0530 Subject: [PATCH 006/344] Added TIMESPAN flags to match archive in various commands (Borg2 feature) Signed-off-by: jetchirag --- borgmatic/actions/mount.py | 1 + borgmatic/actions/prune.py | 1 + borgmatic/borg/mount.py | 7 ++++ borgmatic/borg/prune.py | 5 +++ borgmatic/commands/arguments.py | 68 +++++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+) diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index e2703a57..2b661ec2 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -35,6 +35,7 @@ def run_mount( mount_arguments.paths, mount_arguments.foreground, mount_arguments.options, + mount_arguments, storage, local_borg_version, local_path=local_path, diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index ca098ce4..a89b717f 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -44,6 +44,7 @@ def run_prune( storage, retention, local_borg_version, + prune_arguments, local_path=local_path, remote_path=remote_path, stats=prune_arguments.stats, diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 92d689b2..d987ca4e 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -13,6 +13,7 @@ def mount_archive( paths, foreground, options, + mount_arguments, storage_config, local_borg_version, local_path='borg', @@ -35,6 +36,12 @@ def mount_archive( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--foreground',) if foreground else ()) + + (flags.make_flags('first', mount_arguments.first) if mount_arguments.first else ()) + + (flags.make_flags('last', mount_arguments.last) if mount_arguments.last else ()) + + (flags.make_flags('newest', mount_arguments.newest) if mount_arguments.newest else ()) + + (flags.make_flags('oldest', mount_arguments.oldest) if mount_arguments.oldest else ()) + + (flags.make_flags('older', mount_arguments.older) if mount_arguments.older else ()) + + (flags.make_flags('newer', mount_arguments.newer) if mount_arguments.newer else ()) + (('-o', options) if options else ()) + ( ( diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index e53197f1..aa81a7f9 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -43,6 +43,7 @@ def prune_archives( storage_config, retention_config, local_borg_version, + prune_arguments, local_path='borg', remote_path=None, stats=False, @@ -71,6 +72,10 @@ def prune_archives( + (('--stats',) if stats and not dry_run else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--list',) if list_archives else ()) + + (flags.make_flags('newest', prune_arguments.newest) if prune_arguments.newest else ()) + + (flags.make_flags('oldest', prune_arguments.oldest) if prune_arguments.oldest else ()) + + (flags.make_flags('older', prune_arguments.older) if prune_arguments.older else ()) + + (flags.make_flags('newer', prune_arguments.newer) if prune_arguments.newer else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index d5dc6af4..34ff8043 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -321,6 +321,18 @@ def make_parsers(): transfer_group.add_argument( '--last', metavar='N', help='Only transfer last N archives after other filters are applied' ) + transfer_group.add_argument( + '--oldest', metavar='TIMESPAN', help='Transfer archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + transfer_group.add_argument( + '--newest', metavar='TIMESPAN', help='Transfer archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + transfer_group.add_argument( + '--older', metavar='TIMESPAN', help='Transfer archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) + transfer_group.add_argument( + '--newer', metavar='TIMESPAN', help='Transfer archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) transfer_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' ) @@ -347,6 +359,18 @@ def make_parsers(): prune_group.add_argument( '--list', dest='list_archives', action='store_true', help='List archives kept/pruned' ) + prune_group.add_argument( + '--oldest', metavar='TIMESPAN', help='Consider archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + prune_group.add_argument( + '--newest', metavar='TIMESPAN', help='Consider archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + prune_group.add_argument( + '--older', metavar='TIMESPAN', help='Consider archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) + prune_group.add_argument( + '--newer', metavar='TIMESPAN', help='Consider archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') compact_parser = subparsers.add_parser( @@ -587,6 +611,26 @@ def make_parsers(): action='store_true', help='Stay in foreground until ctrl-C is pressed', ) + mount_group.add_argument( + '--first', + metavar='N', + help='Mount first N archives after other filters are applied', + ) + mount_group.add_argument( + '--last', metavar='N', help='Mount last N archives after other filters are applied' + ) + mount_group.add_argument( + '--oldest', metavar='TIMESPAN', help='Mount archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + mount_group.add_argument( + '--newest', metavar='TIMESPAN', help='Mount archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + mount_group.add_argument( + '--older', metavar='TIMESPAN', help='Mount archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) + mount_group.add_argument( + '--newer', metavar='TIMESPAN', help='Mount archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) mount_group.add_argument('--options', dest='options', help='Extra Borg mount options') mount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') @@ -670,6 +714,18 @@ def make_parsers(): rlist_group.add_argument( '--last', metavar='N', help='List last N archives after other filters are applied' ) + rlist_group.add_argument( + '--oldest', metavar='TIMESPAN', help='List archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + rlist_group.add_argument( + '--newest', metavar='TIMESPAN', help='List archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + rlist_group.add_argument( + '--older', metavar='TIMESPAN', help='List archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) + rlist_group.add_argument( + '--newer', metavar='TIMESPAN', help='List archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) rlist_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') list_parser = subparsers.add_parser( @@ -799,6 +855,18 @@ def make_parsers(): info_group.add_argument( '--last', metavar='N', help='Show info for last N archives after other filters are applied' ) + info_group.add_argument( + '--oldest', metavar='TIMESPAN', help='Show info for archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + info_group.add_argument( + '--newest', metavar='TIMESPAN', help='Show info for archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + ) + info_group.add_argument( + '--older', metavar='TIMESPAN', help='Show info for archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) + info_group.add_argument( + '--newer', metavar='TIMESPAN', help='Show info for archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + ) info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') break_lock_parser = subparsers.add_parser( From ff1f4dc09c9c38def4bfad4f528e5ced4eff779f Mon Sep 17 00:00:00 2001 From: jetchirag Date: Sun, 26 Mar 2023 02:06:46 +0530 Subject: [PATCH 007/344] minor fixes to prune argument help text --- borgmatic/commands/arguments.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 34ff8043..b7ee94aa 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -360,16 +360,16 @@ def make_parsers(): '--list', dest='list_archives', action='store_true', help='List archives kept/pruned' ) prune_group.add_argument( - '--oldest', metavar='TIMESPAN', help='Consider archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + '--oldest', metavar='TIMESPAN', help='Prune archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' ) prune_group.add_argument( - '--newest', metavar='TIMESPAN', help='Consider archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', metavar='TIMESPAN', help='Prune archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' ) prune_group.add_argument( - '--older', metavar='TIMESPAN', help='Consider archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--older', metavar='TIMESPAN', help='Prune archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' ) prune_group.add_argument( - '--newer', metavar='TIMESPAN', help='Consider archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--newer', metavar='TIMESPAN', help='Prune archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' ) prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') From b511e679ae7792257afa7201dfc55314df72bc2e Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sun, 26 Mar 2023 16:59:29 +0530 Subject: [PATCH 008/344] remove optional label for repos from tests --- borgmatic/commands/borgmatic.py | 5 +- borgmatic/config/normalize.py | 19 +--- borgmatic/config/schema.yaml | 2 +- borgmatic/config/validate.py | 2 +- tests/integration/config/test_validate.py | 18 +-- tests/unit/commands/test_borgmatic.py | 130 +++++++++++----------- tests/unit/config/test_normalize.py | 10 +- 7 files changed, 86 insertions(+), 100 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 7612970c..0954ba63 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -107,8 +107,7 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository, retry_num = repo_queue.get() - if isinstance(repository, str): - repository = {'path': repository} + logger.debug(f'{repository["path"]}: Running actions for repository') timeout = retry_num * retry_wait if timeout: logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') @@ -261,8 +260,6 @@ def run_actions( invalid. ''' add_custom_log_levels() - if isinstance(repository, str): - repository = {'path': repository} repository_path = os.path.expanduser(repository['path']) global_arguments = arguments['global'] dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index eb2ed013..bcf088ac 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -80,17 +80,11 @@ def normalize(config_filename, config): updated_repository_path = os.path.abspath( repository_path.partition('file://')[-1] ) - config['location']['repositories'].append( - { - 'path': updated_repository_path, - 'label': repository_dict.get('label', ''), - } + dict(repository_dict, path=updated_repository_path,) ) elif repository_path.startswith('ssh://'): - config['location']['repositories'].append( - {'path': repository_path, 'label': repository_dict.get('label', '')} - ) + config['location']['repositories'].append(repository_dict) else: rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( @@ -103,14 +97,9 @@ def normalize(config_filename, config): ) ) config['location']['repositories'].append( - { - 'path': rewritten_repository_path, - 'label': repository_dict.get('label', ''), - } + dict(repository_dict, path=rewritten_repository_path,) ) else: - config['location']['repositories'].append( - {'path': repository_path, 'label': repository_dict.get('label', '')} - ) + config['location']['repositories'].append(repository_dict) return logs diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index e24b0eca..4a494199 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -53,7 +53,7 @@ properties: type: string description: | Optional label for the repository. This - can be used with the --repository option + can be used with the --repository flag to select a repository to backup to. If not specified, the repository path is used as the label. diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index e40603fe..abcfe3d2 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -138,7 +138,7 @@ def normalize_repository_path(repository): def repositories_match(first, second): ''' Given two repository dicts with keys 'path' (relative and/or absolute), - and 'label', return whether they match. + and 'label', or two repository paths, return whether they match. ''' if isinstance(first, str): first = {'path': first, 'label': first} diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 31a85cd1..67e510ab 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -65,7 +65,7 @@ def test_parse_configuration_transforms_file_into_mapping(): assert config == { 'location': { 'source_directories': ['/home', '/etc'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], + 'repositories': [{'path': 'hostname.borg'}], }, 'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60}, 'consistency': {'checks': [{'name': 'repository'}, {'name': 'archives'}]}, @@ -92,7 +92,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): assert config == { 'location': { 'source_directories': [f'/home/{string.punctuation}'], - 'repositories': [{'path': 'test.borg', 'label': ''}], + 'repositories': [{'path': 'test.borg'}], } } assert logs == [] @@ -154,10 +154,7 @@ def test_parse_configuration_inlines_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': { - 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], - }, + 'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]}, 'retention': {'keep_daily': 7, 'keep_hourly': 24}, } assert logs == [] @@ -191,10 +188,7 @@ def test_parse_configuration_merges_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': { - 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], - }, + 'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]}, 'retention': {'keep_daily': 1, 'keep_hourly': 24}, } assert logs == [] @@ -256,7 +250,7 @@ def test_parse_configuration_applies_overrides(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], + 'repositories': [{'path': 'hostname.borg'}], 'local_path': 'borg2', } } @@ -282,7 +276,7 @@ def test_parse_configuration_applies_normalization(): assert config == { 'location': { 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg', 'label': ''}], + 'repositories': [{'path': 'hostname.borg'}], 'exclude_if_present': ['.nobackup'], } } diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 14e7443e..b9a16f35 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -15,7 +15,7 @@ def test_run_configuration_runs_actions_for_each_repository(): flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return( expected_results[1:] ) - config = {'location': {'repositories': ['foo', 'bar']}} + config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}} arguments = {'global': flexmock(monitoring_verbosity=1)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -75,7 +75,7 @@ def test_run_configuration_logs_actions_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -91,7 +91,7 @@ def test_run_configuration_bails_for_actions_soft_failure(): flexmock(module).should_receive('run_actions').and_raise(error) flexmock(module).should_receive('log_error_records').never() flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -108,7 +108,7 @@ def test_run_configuration_logs_monitor_log_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -126,7 +126,7 @@ def test_run_configuration_bails_for_monitor_log_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -143,7 +143,7 @@ def test_run_configuration_logs_monitor_finish_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -161,7 +161,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -178,7 +178,7 @@ def test_run_configuration_logs_on_error_hook_error(): expected_results[:1] ).and_return(expected_results[1:]) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -194,7 +194,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': ['foo']}} + config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -209,7 +209,7 @@ def test_run_configuration_retries_soft_error(): flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} + config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == [] @@ -231,7 +231,7 @@ def test_run_configuration_retries_hard_error(): flexmock(module).should_receive('log_error_records').with_args( 'foo: Error running actions for repository', OSError, ).and_return(error_logs) - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} + config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -249,41 +249,41 @@ def test_run_configuration_repos_ordered(): flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError ).and_return(expected_results[1:]).ordered() - config = {'location': {'repositories': ['foo', 'bar']}} + config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results -def test_run_configuration_retries_round_robbin(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') - flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) - flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', - OSError, - levelno=logging.WARNING, - log_command_error_output=True, - ).and_return([flexmock()]).ordered() - flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', - OSError, - levelno=logging.WARNING, - log_command_error_output=True, - ).and_return([flexmock()]).ordered() - foo_error_logs = [flexmock()] - flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', OSError - ).and_return(foo_error_logs).ordered() - bar_error_logs = [flexmock()] - flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', OSError - ).and_return(bar_error_logs).ordered() - config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) - assert results == foo_error_logs + bar_error_logs +# def test_run_configuration_retries_round_robin(): +# flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) +# flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) +# flexmock(module.command).should_receive('execute_hook') +# flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) +# flexmock(module).should_receive('log_error_records').with_args( +# 'foo: Error running actions for repository', +# OSError, +# levelno=logging.WARNING, +# log_command_error_output=True, +# ).and_return([flexmock()]).ordered() +# flexmock(module).should_receive('log_error_records').with_args( +# 'bar: Error running actions for repository', +# OSError, +# levelno=logging.WARNING, +# log_command_error_output=True, +# ).and_return([flexmock()]).ordered() +# foo_error_logs = [flexmock()] +# flexmock(module).should_receive('log_error_records').with_args( +# 'foo: Error running actions for repository', OSError +# ).and_return(foo_error_logs).ordered() +# bar_error_logs = [flexmock()] +# flexmock(module).should_receive('log_error_records').with_args( +# 'bar: Error running actions for repository', OSError +# ).and_return(bar_error_logs).ordered() +# config = {'location': {'repositories': [{'path':'foo','path':'bar'}]}, 'storage': {'retries': 1}} +# arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} +# results = list(module.run_configuration('test.yaml', config, arguments)) +# assert results == foo_error_logs + bar_error_logs def test_run_configuration_retries_one_passes(): @@ -309,7 +309,10 @@ def test_run_configuration_retries_one_passes(): flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError ).and_return(error_logs).ordered() - config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} + config = { + 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, + 'storage': {'retries': 1}, + } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -348,7 +351,10 @@ def test_run_configuration_retry_wait(): flexmock(module).should_receive('log_error_records').with_args( 'foo: Error running actions for repository', OSError ).and_return(error_logs).ordered() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}} + config = { + 'location': {'repositories': [{'path': 'foo'}]}, + 'storage': {'retries': 3, 'retry_wait': 10}, + } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -384,7 +390,7 @@ def test_run_configuration_retries_timeout_multiple_repos(): 'bar: Error running actions for repository', OSError ).and_return(error_logs).ordered() config = { - 'location': {'repositories': ['foo', 'bar']}, + 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, 'storage': {'retries': 1, 'retry_wait': 10}, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -409,7 +415,7 @@ def test_run_actions_runs_rcreate(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -431,7 +437,7 @@ def test_run_actions_runs_transfer(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -454,7 +460,7 @@ def test_run_actions_runs_create(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -477,7 +483,7 @@ def test_run_actions_runs_prune(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -499,7 +505,7 @@ def test_run_actions_runs_compact(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -522,7 +528,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -545,7 +551,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -567,7 +573,7 @@ def test_run_actions_runs_extract(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -589,7 +595,7 @@ def test_run_actions_runs_export_tar(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -611,7 +617,7 @@ def test_run_actions_runs_mount(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -633,7 +639,7 @@ def test_run_actions_runs_restore(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -656,7 +662,7 @@ def test_run_actions_runs_rlist(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -680,7 +686,7 @@ def test_run_actions_runs_list(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -704,7 +710,7 @@ def test_run_actions_runs_rinfo(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -728,7 +734,7 @@ def test_run_actions_runs_info(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) assert result == (expected,) @@ -751,7 +757,7 @@ def test_run_actions_runs_break_lock(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -773,7 +779,7 @@ def test_run_actions_runs_borg(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) @@ -800,7 +806,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), - repository='repo', + repository={'path': 'repo'}, ) ) diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index ff67aa12..fe9e7ac1 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -69,27 +69,27 @@ from borgmatic.config import normalize as module ), ( {'location': {'repositories': ['foo@bar:/repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/repo'}]}}, True, ), ( {'location': {'repositories': ['foo@bar:repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/./repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/./repo'}]}}, True, ), ( {'location': {'repositories': ['foo@bar:~/repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo'}]}}, True, ), ( {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo'}]}}, False, ), ( {'location': {'repositories': ['file:///repo']}}, - {'location': {'repositories': [{'path': '/repo', 'label': ''}]}}, + {'location': {'repositories': [{'path': '/repo'}]}}, False, ), ), From a136fda92d599f0dd6ecbf48e82f6bc0771ef956 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sun, 26 Mar 2023 23:35:47 +0530 Subject: [PATCH 009/344] check all tests --- tests/unit/commands/test_borgmatic.py | 61 ++++++++++++++------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index b9a16f35..11e879df 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -255,35 +255,38 @@ def test_run_configuration_repos_ordered(): assert results == expected_results -# def test_run_configuration_retries_round_robin(): -# flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) -# flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) -# flexmock(module.command).should_receive('execute_hook') -# flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) -# flexmock(module).should_receive('log_error_records').with_args( -# 'foo: Error running actions for repository', -# OSError, -# levelno=logging.WARNING, -# log_command_error_output=True, -# ).and_return([flexmock()]).ordered() -# flexmock(module).should_receive('log_error_records').with_args( -# 'bar: Error running actions for repository', -# OSError, -# levelno=logging.WARNING, -# log_command_error_output=True, -# ).and_return([flexmock()]).ordered() -# foo_error_logs = [flexmock()] -# flexmock(module).should_receive('log_error_records').with_args( -# 'foo: Error running actions for repository', OSError -# ).and_return(foo_error_logs).ordered() -# bar_error_logs = [flexmock()] -# flexmock(module).should_receive('log_error_records').with_args( -# 'bar: Error running actions for repository', OSError -# ).and_return(bar_error_logs).ordered() -# config = {'location': {'repositories': [{'path':'foo','path':'bar'}]}, 'storage': {'retries': 1}} -# arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} -# results = list(module.run_configuration('test.yaml', config, arguments)) -# assert results == foo_error_logs + bar_error_logs +def test_run_configuration_retries_round_robin(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) + flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) + flexmock(module).should_receive('log_error_records').with_args( + 'foo: Error running actions for repository', + OSError, + levelno=logging.WARNING, + log_command_error_output=True, + ).and_return([flexmock()]).ordered() + flexmock(module).should_receive('log_error_records').with_args( + 'bar: Error running actions for repository', + OSError, + levelno=logging.WARNING, + log_command_error_output=True, + ).and_return([flexmock()]).ordered() + foo_error_logs = [flexmock()] + flexmock(module).should_receive('log_error_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(foo_error_logs).ordered() + bar_error_logs = [flexmock()] + flexmock(module).should_receive('log_error_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(bar_error_logs).ordered() + config = { + 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, + 'storage': {'retries': 1}, + } + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == foo_error_logs + bar_error_logs def test_run_configuration_retries_one_passes(): From ec9def4e71a98d876fdfdbbbe718b8965452491e Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sun, 26 Mar 2023 23:52:25 +0530 Subject: [PATCH 010/344] rename repository arg to repository_path in all borg actions --- borgmatic/borg/borg.py | 6 +- borgmatic/borg/break_lock.py | 4 +- borgmatic/borg/check.py | 10 ++-- borgmatic/borg/compact.py | 6 +- borgmatic/borg/create.py | 12 ++-- borgmatic/borg/export_tar.py | 6 +- borgmatic/borg/extract.py | 8 ++- borgmatic/borg/flags.py | 10 ++-- borgmatic/borg/info.py | 4 +- borgmatic/borg/list.py | 31 +++++----- borgmatic/borg/mount.py | 8 +-- borgmatic/borg/prune.py | 4 +- borgmatic/borg/rcreate.py | 10 ++-- borgmatic/borg/rinfo.py | 4 +- borgmatic/borg/rlist.py | 24 +++++--- borgmatic/borg/transfer.py | 4 +- tests/unit/borg/test_borg.py | 43 +++++++++----- tests/unit/borg/test_break_lock.py | 12 ++-- tests/unit/borg/test_check.py | 28 ++++----- tests/unit/borg/test_compact.py | 30 ++++++---- tests/unit/borg/test_create.py | 92 +++++++++++++++--------------- tests/unit/borg/test_export_tar.py | 26 ++++----- tests/unit/borg/test_extract.py | 14 ++--- tests/unit/borg/test_flags.py | 10 ++-- tests/unit/borg/test_info.py | 24 ++++---- tests/unit/borg/test_list.py | 58 +++++++++---------- tests/unit/borg/test_mount.py | 22 +++---- tests/unit/borg/test_prune.py | 22 +++---- tests/unit/borg/test_rcreate.py | 30 +++++----- tests/unit/borg/test_rinfo.py | 20 +++---- tests/unit/borg/test_rlist.py | 30 +++++----- tests/unit/borg/test_transfer.py | 24 ++++---- 32 files changed, 339 insertions(+), 297 deletions(-) diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index 460d9d68..f19d6555 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -13,7 +13,7 @@ BORG_SUBCOMMANDS_WITHOUT_REPOSITORY = (('debug', 'info'), ('debug', 'convert-pro def run_arbitrary_borg( - repository, + repository_path, storage_config, local_borg_version, options, @@ -44,10 +44,10 @@ def run_arbitrary_borg( repository_archive_flags = () elif archive: repository_archive_flags = flags.make_repository_archive_flags( - repository, archive, local_borg_version + repository_path, archive, local_borg_version ) else: - repository_archive_flags = flags.make_repository_flags(repository, local_borg_version) + repository_archive_flags = flags.make_repository_flags(repository_path, local_borg_version) full_command = ( (local_path,) diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 820b1c56..2dff31ec 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) def break_lock( - repository, storage_config, local_borg_version, local_path='borg', remote_path=None, + repository_path, storage_config, local_borg_version, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage configuration dict, the local Borg version, @@ -24,7 +24,7 @@ def break_lock( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) borg_environment = environment.make_environment(storage_config) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 2914d83b..993a3c0a 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -243,7 +243,7 @@ def read_check_time(path): def check_archives( - repository, + repository_path, location_config, storage_config, consistency_config, @@ -268,7 +268,7 @@ def check_archives( try: borg_repository_id = json.loads( rinfo.display_repository_info( - repository, + repository_path, storage_config, local_borg_version, argparse.Namespace(json=True), @@ -277,7 +277,7 @@ def check_archives( ) )['repository']['id'] except (json.JSONDecodeError, KeyError): - raise ValueError(f'Cannot determine Borg repository ID for {repository}') + raise ValueError(f'Cannot determine Borg repository ID for {repository_path}') checks = filter_checks_on_frequency( location_config, @@ -310,7 +310,7 @@ def check_archives( + verbosity_flags + (('--progress',) if progress else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) borg_environment = environment.make_environment(storage_config) @@ -329,6 +329,6 @@ def check_archives( if 'extract' in checks: extract.extract_last_archive_dry_run( - storage_config, local_borg_version, repository, lock_wait, local_path, remote_path + storage_config, local_borg_version, repository_path, lock_wait, local_path, remote_path ) write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract')) diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index 847ed26b..0e9d3e89 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def compact_segments( dry_run, - repository, + repository_path, storage_config, local_borg_version, local_path='borg', @@ -36,11 +36,11 @@ def compact_segments( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) if dry_run: - logging.info(f'{repository}: Skipping compact (dry run)') + logging.info(f'{repository_path}: Skipping compact (dry run)') return execute_command( diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index d557a6ab..58c3509b 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -322,7 +322,7 @@ def check_all_source_directories_exist(source_directories): def create_archive( dry_run, - repository, + repository_path, location_config, storage_config, local_borg_version, @@ -411,7 +411,7 @@ def create_archive( if stream_processes and location_config.get('read_special') is False: logger.warning( - f'{repository}: Ignoring configured "read_special" value of false, as true is needed for database hooks.' + f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.' ) create_command = ( @@ -446,7 +446,9 @@ def create_archive( ) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_archive_flags(repository, archive_name_format, local_borg_version) + + flags.make_repository_archive_flags( + repository_path, archive_name_format, local_borg_version + ) + (sources if not pattern_file else ()) ) @@ -466,7 +468,7 @@ def create_archive( # If database hooks are enabled (as indicated by streaming processes), exclude files that might # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True. if stream_processes and not location_config.get('read_special'): - logger.debug(f'{repository}: Collecting special file paths') + logger.debug(f'{repository_path}: Collecting special file paths') special_file_paths = collect_special_file_paths( create_command, local_path, @@ -477,7 +479,7 @@ def create_archive( if special_file_paths: logger.warning( - f'{repository}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}' + f'{repository_path}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}' ) exclude_file = write_pattern_file( expand_home_directories( diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 01d1b7ed..2b427f03 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def export_tar_archive( dry_run, - repository, + repository_path, archive, paths, destination_path, @@ -45,7 +45,7 @@ def export_tar_archive( + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + flags.make_repository_archive_flags(repository, archive, local_borg_version,) + + flags.make_repository_archive_flags(repository_path, archive, local_borg_version,) + (destination_path,) + (tuple(paths) if paths else ()) ) @@ -56,7 +56,7 @@ def export_tar_archive( output_log_level = logging.INFO if dry_run: - logging.info(f'{repository}: Skipping export to tar file (dry run)') + logging.info(f'{repository_path}: Skipping export to tar file (dry run)') return execute_command( diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 6c32f7f0..a95685cf 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) def extract_last_archive_dry_run( storage_config, local_borg_version, - repository, + repository_path, lock_wait=None, local_path='borg', remote_path=None, @@ -30,7 +30,7 @@ def extract_last_archive_dry_run( try: last_archive_name = rlist.resolve_archive_name( - repository, 'latest', storage_config, local_borg_version, local_path, remote_path + repository_path, 'latest', storage_config, local_borg_version, local_path, remote_path ) except ValueError: logger.warning('No archives found. Skipping extract consistency check.') @@ -44,7 +44,9 @@ def extract_last_archive_dry_run( + lock_wait_flags + verbosity_flags + list_flag - + flags.make_repository_archive_flags(repository, last_archive_name, local_borg_version) + + flags.make_repository_archive_flags( + repository_path, last_archive_name, local_borg_version + ) ) execute_command( diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 845e0ff3..5dcebf50 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -33,7 +33,7 @@ def make_flags_from_arguments(arguments, excludes=()): ) -def make_repository_flags(repository, local_borg_version): +def make_repository_flags(repository_path, local_borg_version): ''' Given the path of a Borg repository and the local Borg version, return Borg-version-appropriate command-line flags (as a tuple) for selecting that repository. @@ -42,17 +42,17 @@ def make_repository_flags(repository, local_borg_version): ('--repo',) if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) else () - ) + (repository,) + ) + (repository_path,) -def make_repository_archive_flags(repository, archive, local_borg_version): +def make_repository_archive_flags(repository_path, archive, local_borg_version): ''' Given the path of a Borg repository, an archive name or pattern, and the local Borg version, return Borg-version-appropriate command-line flags (as a tuple) for selecting that repository and archive. ''' return ( - ('--repo', repository, archive) + ('--repo', repository_path, archive) if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) - else (f'{repository}::{archive}',) + else (f'{repository_path}::{archive}',) ) diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index bcde24cd..6142104f 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def display_archives_info( - repository, + repository_path, storage_config, local_borg_version, info_arguments, @@ -49,7 +49,7 @@ def display_archives_info( + flags.make_flags_from_arguments( info_arguments, excludes=('repository', 'archive', 'prefix') ) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) + ( flags.make_flags('match-archives', info_arguments.archive) if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 916d17b0..908f8fef 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -21,7 +21,7 @@ MAKE_FLAGS_EXCLUDES = ( def make_list_command( - repository, + repository_path, storage_config, local_borg_version, list_arguments, @@ -52,10 +52,10 @@ def make_list_command( + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES) + ( flags.make_repository_archive_flags( - repository, list_arguments.archive, local_borg_version + repository_path, list_arguments.archive, local_borg_version ) if list_arguments.archive - else flags.make_repository_flags(repository, local_borg_version) + else flags.make_repository_flags(repository_path, local_borg_version) ) + (tuple(list_arguments.paths) if list_arguments.paths else ()) ) @@ -86,7 +86,7 @@ def make_find_paths(find_paths): def capture_archive_listing( - repository, + repository_path, archive, storage_config, local_borg_version, @@ -104,11 +104,11 @@ def capture_archive_listing( return tuple( execute_command_and_capture_output( make_list_command( - repository, + repository_path, storage_config, local_borg_version, argparse.Namespace( - repository=repository, + repository=repository_path, archive=archive, paths=[f'sh:{list_path}'], find_paths=None, @@ -126,7 +126,7 @@ def capture_archive_listing( def list_archive( - repository, + repository_path, storage_config, local_borg_version, list_arguments, @@ -149,7 +149,7 @@ def list_archive( ) rlist_arguments = argparse.Namespace( - repository=repository, + repository=repository_path, short=list_arguments.short, format=list_arguments.format, json=list_arguments.json, @@ -160,7 +160,12 @@ def list_archive( last=list_arguments.last, ) return rlist.list_repository( - repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path + repository_path, + storage_config, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ) if list_arguments.archive: @@ -181,7 +186,7 @@ def list_archive( # getting a list of archives to search. if list_arguments.find_paths and not list_arguments.archive: rlist_arguments = argparse.Namespace( - repository=repository, + repository=repository_path, short=True, format=None, json=None, @@ -196,7 +201,7 @@ def list_archive( archive_lines = tuple( execute_command_and_capture_output( rlist.make_rlist_command( - repository, + repository_path, storage_config, local_borg_version, rlist_arguments, @@ -213,7 +218,7 @@ def list_archive( # For each archive listed by Borg, run list on the contents of that archive. for archive in archive_lines: - logger.answer(f'{repository}: Listing archive {archive}') + logger.answer(f'{repository_path}: Listing archive {archive}') archive_arguments = copy.copy(list_arguments) archive_arguments.archive = archive @@ -224,7 +229,7 @@ def list_archive( setattr(archive_arguments, name, None) main_command = make_list_command( - repository, + repository_path, storage_config, local_borg_version, archive_arguments, diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 92d689b2..07a6c632 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) def mount_archive( - repository, + repository_path, archive, mount_point, paths, @@ -38,7 +38,7 @@ def mount_archive( + (('-o', options) if options else ()) + ( ( - flags.make_repository_flags(repository, local_borg_version) + flags.make_repository_flags(repository_path, local_borg_version) + ( ('--match-archives', archive) if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) @@ -47,9 +47,9 @@ def mount_archive( ) if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) else ( - flags.make_repository_archive_flags(repository, archive, local_borg_version) + flags.make_repository_archive_flags(repository_path, archive, local_borg_version) if archive - else flags.make_repository_flags(repository, local_borg_version) + else flags.make_repository_flags(repository_path, local_borg_version) ) ) + (mount_point,) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 5be85de2..d21ceee3 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -39,7 +39,7 @@ def make_prune_flags(retention_config, local_borg_version): def prune_archives( dry_run, - repository, + repository_path, storage_config, retention_config, local_borg_version, @@ -74,7 +74,7 @@ def prune_archives( + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) if stats or list_archives: diff --git a/borgmatic/borg/rcreate.py b/borgmatic/borg/rcreate.py index d3a8f7aa..7510529d 100644 --- a/borgmatic/borg/rcreate.py +++ b/borgmatic/borg/rcreate.py @@ -13,7 +13,7 @@ RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2 def create_repository( dry_run, - repository, + repository_path, storage_config, local_borg_version, encryption_mode, @@ -33,14 +33,14 @@ def create_repository( ''' try: rinfo.display_repository_info( - repository, + repository_path, storage_config, local_borg_version, argparse.Namespace(json=True), local_path, remote_path, ) - logger.info(f'{repository}: Repository already exists. Skipping creation.') + logger.info(f'{repository_path}: Repository already exists. Skipping creation.') return except subprocess.CalledProcessError as error: if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE: @@ -65,11 +65,11 @@ def create_repository( + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + (('--remote-path', remote_path) if remote_path else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) if dry_run: - logging.info(f'{repository}: Skipping repository creation (dry run)') + logging.info(f'{repository_path}: Skipping repository creation (dry run)') return # Do not capture output here, so as to support interactive prompts. diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index 7bc9a5e9..8a30b7f0 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def display_repository_info( - repository, + repository_path, storage_config, local_borg_version, rinfo_arguments, @@ -43,7 +43,7 @@ def display_repository_info( + flags.make_flags('remote-path', remote_path) + flags.make_flags('lock-wait', lock_wait) + (('--json',) if rinfo_arguments.json else ()) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) extra_environment = environment.make_environment(storage_config) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 43bc28d6..8625363b 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def resolve_archive_name( - repository, archive, storage_config, local_borg_version, local_path='borg', remote_path=None + repository_path, + archive, + storage_config, + local_borg_version, + local_path='borg', + remote_path=None, ): ''' Given a local or remote repository path, an archive name, a storage config dict, a local Borg @@ -31,7 +36,7 @@ def resolve_archive_name( + flags.make_flags('lock-wait', lock_wait) + flags.make_flags('last', 1) + ('--short',) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) output = execute_command_and_capture_output( @@ -42,7 +47,7 @@ def resolve_archive_name( except IndexError: raise ValueError('No archives found in the repository') - logger.debug(f'{repository}: Latest archive is {latest_archive}') + logger.debug(f'{repository_path}: Latest archive is {latest_archive}') return latest_archive @@ -51,7 +56,7 @@ MAKE_FLAGS_EXCLUDES = ('repository', 'prefix') def make_rlist_command( - repository, + repository_path, storage_config, local_borg_version, rlist_arguments, @@ -92,12 +97,12 @@ def make_rlist_command( else () ) + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) ) def list_repository( - repository, + repository_path, storage_config, local_borg_version, rlist_arguments, @@ -113,7 +118,12 @@ def list_repository( borg_environment = environment.make_environment(storage_config) main_command = make_rlist_command( - repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path + repository_path, + storage_config, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ) if rlist_arguments.json: diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index bad02d06..29e205c7 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def transfer_archives( dry_run, - repository, + repository_path, storage_config, local_borg_version, transfer_arguments, @@ -38,7 +38,7 @@ def transfer_archives( transfer_arguments, excludes=('repository', 'source_repository', 'archive', 'match_archives'), ) - + flags.make_repository_flags(repository, local_borg_version) + + flags.make_repository_flags(repository_path, local_borg_version) + flags.make_flags('other-repo', transfer_arguments.source_repository) + flags.make_flags('dry-run', dry_run) ) diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 6eae5fb4..545da6c0 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -21,7 +21,10 @@ def test_run_arbitrary_borg_calls_borg_with_parameters(): ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['break-lock'], ) @@ -40,7 +43,10 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['break-lock'], ) @@ -59,7 +65,10 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['break-lock'], ) @@ -80,7 +89,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters( ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', options=['break-lock'], @@ -103,7 +112,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], @@ -125,7 +134,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], @@ -149,7 +158,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['break-lock'], @@ -171,7 +180,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['list', '--progress'], @@ -192,7 +201,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['--', 'break-lock'], @@ -213,7 +222,7 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=[], + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=[], ) @@ -231,7 +240,10 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository(): ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['key', 'export'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['key', 'export'], ) @@ -249,7 +261,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository() ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['debug', 'dump-manifest', 'path'], @@ -270,7 +282,10 @@ def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repositor ) module.run_arbitrary_borg( - repository='repo', storage_config={}, local_borg_version='1.2.3', options=['debug', 'info'], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=['debug', 'info'], ) @@ -288,7 +303,7 @@ def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_bor ) module.run_arbitrary_borg( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=['debug', 'convert-profile', 'in', 'out'], diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 0663f93c..210aceeb 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -19,7 +19,7 @@ def test_break_lock_calls_borg_with_required_flags(): insert_execute_command_mock(('borg', 'break-lock', 'repo')) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', ) @@ -28,7 +28,7 @@ def test_break_lock_calls_borg_with_remote_path_flags(): insert_execute_command_mock(('borg', 'break-lock', '--remote-path', 'borg1', 'repo')) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', ) @@ -37,7 +37,7 @@ def test_break_lock_calls_borg_with_umask_flags(): insert_execute_command_mock(('borg', 'break-lock', '--umask', '0770', 'repo')) module.break_lock( - repository='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + repository_path='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', ) @@ -46,7 +46,7 @@ def test_break_lock_calls_borg_with_lock_wait_flags(): insert_execute_command_mock(('borg', 'break-lock', '--lock-wait', '5', 'repo')) module.break_lock( - repository='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + repository_path='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', ) @@ -56,7 +56,7 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', ) @@ -66,5 +66,5 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags(): insert_logging_mock(logging.DEBUG) module.break_lock( - repository='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index ba82225c..75755565 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -370,7 +370,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -400,7 +400,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -435,7 +435,7 @@ def test_check_archives_calls_borg_with_parameters(checks): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -455,7 +455,7 @@ def test_check_archives_with_json_error_raises(): with pytest.raises(ValueError): module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -473,7 +473,7 @@ def test_check_archives_with_missing_json_keys_raises(): with pytest.raises(ValueError): module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -497,7 +497,7 @@ def test_check_archives_with_extract_check_calls_extract_only(): insert_execute_command_never() module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -521,7 +521,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -545,7 +545,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -563,7 +563,7 @@ def test_check_archives_without_any_checks_bails(): insert_execute_command_never() module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -589,7 +589,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -616,7 +616,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -643,7 +643,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={'lock_wait': 5}, consistency_config=consistency_config, @@ -670,7 +670,7 @@ def test_check_archives_with_retention_prefix(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={}, consistency_config=consistency_config, @@ -693,7 +693,7 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): flexmock(module).should_receive('write_check_time') module.check_archives( - repository='repo', + repository_path='repo', location_config={}, storage_config={'extra_borg_options': {'check': '--extra --options'}}, consistency_config=consistency_config, diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 36760f3c..60447db6 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -25,7 +25,7 @@ def test_compact_segments_calls_borg_with_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) module.compact_segments( - dry_run=False, repository='repo', storage_config={}, local_borg_version='1.2.3' + dry_run=False, repository_path='repo', storage_config={}, local_borg_version='1.2.3' ) @@ -35,7 +35,7 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.compact_segments( - repository='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False ) @@ -45,7 +45,7 @@ def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.compact_segments( - repository='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False ) @@ -53,7 +53,7 @@ def test_compact_segments_with_dry_run_skips_borg_call(): flexmock(module).should_receive('execute_command').never() module.compact_segments( - repository='repo', storage_config={}, local_borg_version='1.2.3', dry_run=True + repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=True ) @@ -63,7 +63,7 @@ def test_compact_segments_with_local_path_calls_borg_via_local_path(): module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', local_path='borg1', @@ -76,7 +76,7 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', @@ -89,7 +89,7 @@ def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', progress=True, @@ -102,7 +102,7 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', cleanup_commits=True, @@ -115,7 +115,7 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', threshold=20, @@ -128,7 +128,10 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.INFO) module.compact_segments( - dry_run=False, repository='repo', storage_config=storage_config, local_borg_version='1.2.3' + dry_run=False, + repository_path='repo', + storage_config=storage_config, + local_borg_version='1.2.3', ) @@ -138,7 +141,10 @@ def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) module.compact_segments( - dry_run=False, repository='repo', storage_config=storage_config, local_borg_version='1.2.3' + dry_run=False, + repository_path='repo', + storage_config=storage_config, + local_borg_version='1.2.3', ) @@ -148,7 +154,7 @@ def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options( module.compact_segments( dry_run=False, - repository='repo', + repository_path='repo', storage_config={'extra_borg_options': {'compact': '--extra --options'}}, local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 5fb51f3c..42fedd56 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -484,7 +484,7 @@ def test_create_archive_calls_borg_with_parameters(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -527,7 +527,7 @@ def test_create_archive_calls_borg_with_environment(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -572,7 +572,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -617,7 +617,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -660,7 +660,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -700,7 +700,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -744,7 +744,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -784,7 +784,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -827,7 +827,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): module.create_archive( dry_run=True, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -872,7 +872,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete module.create_archive( dry_run=True, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -915,7 +915,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -957,7 +957,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -999,7 +999,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1041,7 +1041,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1088,7 +1088,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1132,7 +1132,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1175,7 +1175,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1223,7 +1223,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1276,7 +1276,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1326,7 +1326,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1380,7 +1380,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1434,7 +1434,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1477,7 +1477,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1520,7 +1520,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1563,7 +1563,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1606,7 +1606,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1648,7 +1648,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1690,7 +1690,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1733,7 +1733,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1777,7 +1777,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1820,7 +1820,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1880,7 +1880,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -1943,7 +1943,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2011,7 +2011,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2074,7 +2074,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2115,7 +2115,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): json_output = module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2157,7 +2157,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() json_output = module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2204,7 +2204,7 @@ def test_create_archive_with_source_directories_glob_expands(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo*'], 'repositories': ['repo'], @@ -2247,7 +2247,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo*'], 'repositories': ['repo'], @@ -2289,7 +2289,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo*'], 'repositories': ['repo'], @@ -2331,7 +2331,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2374,7 +2374,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2417,7 +2417,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): module.create_archive( dry_run=False, - repository='{fqdn}', # noqa: FS003 + repository_path='{fqdn}', # noqa: FS003 location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['{fqdn}'], # noqa: FS003 @@ -2459,7 +2459,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2519,7 +2519,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], @@ -2543,7 +2543,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ with pytest.raises(ValueError): module.create_archive( dry_run=False, - repository='repo', + repository_path='repo', location_config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index a3c03526..92776dd4 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -32,7 +32,7 @@ def test_export_tar_archive_calls_borg_with_path_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=['path1', 'path2'], destination_path='test.tar', @@ -53,7 +53,7 @@ def test_export_tar_archive_calls_borg_with_local_path_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -75,7 +75,7 @@ def test_export_tar_archive_calls_borg_with_remote_path_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -97,7 +97,7 @@ def test_export_tar_archive_calls_borg_with_umask_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -118,7 +118,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -138,7 +138,7 @@ def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -160,7 +160,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -179,7 +179,7 @@ def test_export_tar_archive_calls_borg_with_dry_run_parameter(): module.export_tar_archive( dry_run=True, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -200,7 +200,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_parameters(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -223,7 +223,7 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -245,7 +245,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='test.tar', @@ -265,7 +265,7 @@ def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): module.export_tar_archive( dry_run=False, - repository='server:repo', + repository_path='server:repo', archive='archive', paths=None, destination_path='test.tar', @@ -284,7 +284,7 @@ def test_export_tar_archive_calls_borg_with_stdout_destination_path(): module.export_tar_archive( dry_run=False, - repository='repo', + repository_path='repo', archive='archive', paths=None, destination_path='-', diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index d27026e4..de29cf4d 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -23,7 +23,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_last_archive(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -32,7 +32,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_not_raise(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(('repo',)) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -45,7 +45,7 @@ def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_paramet ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -60,7 +60,7 @@ def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_param ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None ) @@ -74,7 +74,7 @@ def test_extract_last_archive_dry_run_calls_borg_via_local_path(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', - repository='repo', + repository_path='repo', lock_wait=None, local_path='borg1', ) @@ -92,7 +92,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', - repository='repo', + repository_path='repo', lock_wait=None, remote_path='borg1', ) @@ -108,7 +108,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=5 + storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=5 ) diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 7bf621da..1985d816 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -50,7 +50,7 @@ def test_make_flags_from_arguments_omits_excludes(): def test_make_repository_flags_with_borg_features_includes_repo_flag(): flexmock(module.feature).should_receive('available').and_return(True) - assert module.make_repository_flags(repository='repo', local_borg_version='1.2.3') == ( + assert module.make_repository_flags(repository_path='repo', local_borg_version='1.2.3') == ( '--repo', 'repo', ) @@ -59,14 +59,16 @@ def test_make_repository_flags_with_borg_features_includes_repo_flag(): def test_make_repository_flags_without_borg_features_includes_omits_flag(): flexmock(module.feature).should_receive('available').and_return(False) - assert module.make_repository_flags(repository='repo', local_borg_version='1.2.3') == ('repo',) + assert module.make_repository_flags(repository_path='repo', local_borg_version='1.2.3') == ( + 'repo', + ) def test_make_repository_archive_flags_with_borg_features_separates_repository_and_archive(): flexmock(module.feature).should_receive('available').and_return(True) assert module.make_repository_archive_flags( - repository='repo', archive='archive', local_borg_version='1.2.3' + repository_path='repo', archive='archive', local_borg_version='1.2.3' ) == ('--repo', 'repo', 'archive',) @@ -74,5 +76,5 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a flexmock(module.feature).should_receive('available').and_return(False) assert module.make_repository_archive_flags( - repository='repo', archive='archive', local_borg_version='1.2.3' + repository_path='repo', archive='archive', local_borg_version='1.2.3' ) == ('repo::archive',) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index ab92065b..fcc556a4 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -23,7 +23,7 @@ def test_display_archives_info_calls_borg_with_parameters(): ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -45,7 +45,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): ) insert_logging_mock(logging.INFO) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -65,7 +65,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu insert_logging_mock(logging.INFO) json_output = module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=True, prefix=None), @@ -90,7 +90,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -110,7 +110,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp insert_logging_mock(logging.DEBUG) json_output = module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=True, prefix=None), @@ -131,7 +131,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): ).and_return('[]') json_output = module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=True, prefix=None), @@ -158,7 +158,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive='archive', json=False, prefix=None), @@ -180,7 +180,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -206,7 +206,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -233,7 +233,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None), @@ -258,7 +258,7 @@ def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parame ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix='foo'), @@ -284,7 +284,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): ) module.display_archives_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', info_arguments=flexmock(archive=None, json=False, prefix=None, **{argument_name: 'value'}), diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index cedcec84..37af65a3 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -16,7 +16,7 @@ def test_make_list_command_includes_log_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -32,7 +32,7 @@ def test_make_list_command_includes_json_but_not_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), @@ -48,7 +48,7 @@ def test_make_list_command_includes_log_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -64,7 +64,7 @@ def test_make_list_command_includes_json_but_not_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), @@ -79,7 +79,7 @@ def test_make_list_command_includes_json(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), @@ -96,7 +96,7 @@ def test_make_list_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={'lock_wait': 5}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -113,7 +113,7 @@ def test_make_list_command_includes_archive(): ) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=None, json=False), @@ -130,7 +130,7 @@ def test_make_list_command_includes_archive_and_path(): ) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False), @@ -145,7 +145,7 @@ def test_make_list_command_includes_local_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -163,7 +163,7 @@ def test_make_list_command_includes_remote_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), @@ -179,7 +179,7 @@ def test_make_list_command_includes_short(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False, short=True), @@ -210,7 +210,7 @@ def test_make_list_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock( @@ -259,7 +259,7 @@ def test_capture_archive_listing_does_not_raise(): flexmock(module).should_receive('make_list_command') module.capture_archive_listing( - repository='repo', + repository_path='repo', archive='archive', storage_config=flexmock(), local_borg_version=flexmock(), @@ -284,7 +284,7 @@ def test_list_archive_calls_borg_with_parameters(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -301,7 +301,7 @@ def test_list_archive_calls_borg_with_parameters(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -318,7 +318,7 @@ def test_list_archive_with_archive_and_json_errors(): with pytest.raises(ValueError): module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -343,7 +343,7 @@ def test_list_archive_calls_borg_with_local_path(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -360,7 +360,7 @@ def test_list_archive_calls_borg_with_local_path(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -408,7 +408,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -433,7 +433,7 @@ def test_list_archive_calls_borg_with_archive(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -450,7 +450,7 @@ def test_list_archive_calls_borg_with_archive(): ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -480,7 +480,7 @@ def test_list_archive_without_archive_delegates_to_list_repository(): flexmock(module).should_receive('execute_command').never() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -510,7 +510,7 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos flexmock(module).should_receive('execute_command').never() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, @@ -537,7 +537,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_fl module.feature.Feature.RLIST, '1.2.3' ).and_return(False) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -556,7 +556,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_fl ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -586,7 +586,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.rlist).should_receive('make_rlist_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=argparse.Namespace( @@ -601,7 +601,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -619,7 +619,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).and_return(('borg', 'list', '--repo', 'repo', 'archive1')) flexmock(module).should_receive('make_list_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( @@ -652,7 +652,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).once() module.list_archive( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 7f2060fa..6161a249 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -20,7 +20,7 @@ def test_mount_archive_calls_borg_with_required_flags(): insert_execute_command_mock(('borg', 'mount', 'repo', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive=None, mount_point='/mnt', paths=None, @@ -39,7 +39,7 @@ def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_a ) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -58,7 +58,7 @@ def test_mount_archive_without_archive_calls_borg_with_repository_flags_only(): insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -77,7 +77,7 @@ def test_mount_archive_calls_borg_with_path_flags(): insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt', 'path1', 'path2')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=['path1', 'path2'], @@ -98,7 +98,7 @@ def test_mount_archive_calls_borg_with_remote_path_flags(): ) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -118,7 +118,7 @@ def test_mount_archive_calls_borg_with_umask_flags(): insert_execute_command_mock(('borg', 'mount', '--umask', '0770', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -137,7 +137,7 @@ def test_mount_archive_calls_borg_with_lock_wait_flags(): insert_execute_command_mock(('borg', 'mount', '--lock-wait', '5', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -157,7 +157,7 @@ def test_mount_archive_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -177,7 +177,7 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_flags(): insert_logging_mock(logging.DEBUG) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -202,7 +202,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): ).once() module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, @@ -221,7 +221,7 @@ def test_mount_archive_calls_borg_with_options_flags(): insert_execute_command_mock(('borg', 'mount', '-o', 'super_mount', 'repo::archive', '/mnt')) module.mount_archive( - repository='repo', + repository_path='repo', archive='archive', mount_point='/mnt', paths=None, diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index dd240dcb..f33c9933 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -98,7 +98,7 @@ def test_prune_archives_calls_borg_with_parameters(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -114,7 +114,7 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.prune_archives( - repository='repo', + repository_path='repo', storage_config={}, dry_run=False, retention_config=flexmock(), @@ -131,7 +131,7 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.prune_archives( - repository='repo', + repository_path='repo', storage_config={}, dry_run=False, retention_config=flexmock(), @@ -147,7 +147,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) module.prune_archives( - repository='repo', + repository_path='repo', storage_config={}, dry_run=True, retention_config=flexmock(), @@ -164,7 +164,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -181,7 +181,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters( module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -198,7 +198,7 @@ def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_ou module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -215,7 +215,7 @@ def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_out module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', @@ -233,7 +233,7 @@ def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', @@ -250,7 +250,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', @@ -266,7 +266,7 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): module.prune_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={'extra_borg_options': {'prune': '--extra --options'}}, retention_config=flexmock(), local_borg_version='1.2.3', diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index 612ec11c..e232df22 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -40,7 +40,7 @@ def test_create_repository_calls_borg_with_flags(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -55,7 +55,7 @@ def test_create_repository_with_dry_run_skips_borg_call(): module.create_repository( dry_run=True, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -74,7 +74,7 @@ def test_create_repository_raises_for_borg_rcreate_error(): with pytest.raises(subprocess.CalledProcessError): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -88,7 +88,7 @@ def test_create_repository_skips_creation_when_repository_already_exists(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -103,7 +103,7 @@ def test_create_repository_raises_for_unknown_rinfo_command_error(): with pytest.raises(subprocess.CalledProcessError): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -118,7 +118,7 @@ def test_create_repository_with_source_repository_calls_borg_with_other_repo_fla module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -134,7 +134,7 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -150,7 +150,7 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -166,7 +166,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -182,7 +182,7 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -199,7 +199,7 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -215,7 +215,7 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -230,7 +230,7 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -246,7 +246,7 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey', @@ -262,7 +262,7 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options module.create_repository( dry_run=False, - repository='repo', + repository_path='repo', storage_config={'extra_borg_options': {'rcreate': '--extra --options'}}, local_borg_version='2.3.4', encryption_mode='repokey', diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index d9cbf9bc..ec0819cf 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -21,7 +21,7 @@ def test_display_repository_info_calls_borg_with_parameters(): ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -42,7 +42,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -63,7 +63,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): ) insert_logging_mock(logging.INFO) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -82,7 +82,7 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out insert_logging_mock(logging.INFO) json_output = module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), @@ -106,7 +106,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter( insert_logging_mock(logging.DEBUG) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -125,7 +125,7 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou insert_logging_mock(logging.DEBUG) json_output = module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), @@ -145,7 +145,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_parameter(): ).and_return('[]') json_output = module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), @@ -168,7 +168,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -190,7 +190,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_pa ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), @@ -213,7 +213,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parame ) module.display_repository_info( - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 178a820f..a098cafa 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -131,7 +131,7 @@ def test_make_rlist_command_includes_log_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -147,7 +147,7 @@ def test_make_rlist_command_includes_json_but_not_info(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), @@ -163,7 +163,7 @@ def test_make_rlist_command_includes_log_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -179,7 +179,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), @@ -194,7 +194,7 @@ def test_make_rlist_command_includes_json(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), @@ -211,7 +211,7 @@ def test_make_rlist_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={'lock_wait': 5}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -226,7 +226,7 @@ def test_make_rlist_command_includes_local_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -244,7 +244,7 @@ def test_make_rlist_command_includes_remote_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), @@ -262,7 +262,7 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), @@ -277,7 +277,7 @@ def test_make_rlist_command_includes_short(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None, short=True), @@ -307,7 +307,7 @@ def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_rlist_command( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( @@ -331,7 +331,7 @@ def test_list_repository_calls_borg_with_parameters(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, @@ -347,7 +347,7 @@ def test_list_repository_calls_borg_with_parameters(): ).once() module.list_repository( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, @@ -362,7 +362,7 @@ def test_list_repository_with_json_returns_borg_output(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, @@ -374,7 +374,7 @@ def test_list_repository_with_json_returns_borg_output(): assert ( module.list_repository( - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 27b03470..a4814420 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -25,7 +25,7 @@ def test_transfer_archives_calls_borg_with_flags(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -54,7 +54,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): module.transfer_archives( dry_run=True, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -80,7 +80,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): insert_logging_mock(logging.INFO) module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -107,7 +107,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -136,7 +136,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -165,7 +165,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -191,7 +191,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -221,7 +221,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -252,7 +252,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -278,7 +278,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -308,7 +308,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( @@ -340,7 +340,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla module.transfer_archives( dry_run=False, - repository='repo', + repository_path='repo', storage_config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( From b336b9bedfc96532863cf2ace0af0a57a7337af2 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 27 Mar 2023 00:19:23 +0530 Subject: [PATCH 011/344] add tests for repo labels --- tests/unit/config/test_normalize.py | 27 +++++++++++++++++++++++++++ tests/unit/config/test_validate.py | 10 ++++++++++ 2 files changed, 37 insertions(+) diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index fe9e7ac1..1d61a771 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -92,6 +92,21 @@ from borgmatic.config import normalize as module {'location': {'repositories': [{'path': '/repo'}]}}, False, ), + ( + {'location': {'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}]}}, + {'location': {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': 'foo'}]}}, + True, + ), + ( + {'location': {'repositories': [{'path': 'file:///repo', 'label': 'foo'}]}}, + {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, + False, + ), + ( + {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, + {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, + False, + ), ), ) def test_normalize_applies_hard_coded_normalization_to_config( @@ -105,3 +120,15 @@ def test_normalize_applies_hard_coded_normalization_to_config( assert logs else: assert logs == [] + + +def test_normalize_raises_error_if_repository_data_is_not_consistent(): + with pytest.raises(TypeError): + module.normalize( + 'test.yaml', + { + 'location': { + 'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}, 'file:///repo'] + } + }, + ) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 28527226..327a2b44 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -121,6 +121,16 @@ def test_guard_configuration_contains_repository_does_not_raise_when_repository_ ) +def test_guard_configuration_contains_repository_does_not_raise_when_repository_label_in_config(): + + module.guard_configuration_contains_repository( + repository='repo', + configurations={ + 'config.yaml': {'location': {'repositories': [{'path': 'foo/bar', 'label': 'repo'}]}} + }, + ) + + def test_guard_configuration_contains_repository_does_not_raise_when_repository_not_given(): module.guard_configuration_contains_repository( repository=None, configurations={'config.yaml': {'location': {'repositories': ['repo']}}} From acbbd6670a154c79f512894c19f9e6e152544cbd Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Mar 2023 21:26:35 -0700 Subject: [PATCH 012/344] Removing debugging command output. --- borgmatic/execute.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 53d1a098..46dd04f3 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -235,12 +235,10 @@ def execute_command_and_capture_output( env=environment, cwd=working_directory, ) - logger.warning(f'Command output: {output}') except subprocess.CalledProcessError as error: if exit_code_indicates_error(command, error.returncode): raise output = error.output - logger.warning(f'Command output: {output}') return output.decode() if output is not None else None From 8bef1c698b3f2c3cdf2d9c3a44c4bed5a9068282 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 27 Mar 2023 22:16:39 +0530 Subject: [PATCH 013/344] add feature to docs --- docs/how-to/backup-your-databases.md | 3 ++- docs/how-to/extract-a-backup.md | 3 ++- docs/how-to/run-arbitrary-borg-commands.md | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index bc21b659..29bf4772 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -231,7 +231,8 @@ If you have a single repository in your borgmatic configuration file(s), no problem: the `restore` action figures out which repository to use. But if you have multiple repositories configured, then you'll need to specify -the repository path containing the archive to restore. Here's an example: +the repository to use via the `--repository` flag. This can be done either +with the repository's path or its label as configured in your borgmatic configuration file. ```bash borgmatic restore --repository repo.borg --archive host-2023-... diff --git a/docs/how-to/extract-a-backup.md b/docs/how-to/extract-a-backup.md index 4285c784..164fc135 100644 --- a/docs/how-to/extract-a-backup.md +++ b/docs/how-to/extract-a-backup.md @@ -51,7 +51,8 @@ If you have a single repository in your borgmatic configuration file(s), no problem: the `extract` action figures out which repository to use. But if you have multiple repositories configured, then you'll need to specify -the repository path containing the archive to extract. Here's an example: +the repository to use via the `--repository` flag. This can be done either +with the repository's path or its label as configured in your borgmatic configuration file. ```bash borgmatic extract --repository repo.borg --archive host-2023-... diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index 3d119b4c..0777ebba 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -53,7 +53,8 @@ This runs Borg's `rlist` command once on each configured borgmatic repository. (The native `borgmatic rlist` action should be preferred for most use.) What if you only want to run Borg on a single configured borgmatic repository -when you've got several configured? Not a problem. +when you've got several configured? Not a problem. The `--repository` argument +lets you specify the repository to use, either by its path or its label: ```bash borgmatic borg --repository repo.borg break-lock From d0d3a398332b89966e57c3617a6acaa71add85bc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 10:36:39 -0700 Subject: [PATCH 014/344] When a database command errors, display and log the error message instead of swallowing it (#396). --- NEWS | 1 + borgmatic/execute.py | 44 ++++++++++++++++++++++++------- borgmatic/hooks/mysql.py | 6 ++--- tests/integration/test_execute.py | 3 +-- tests/unit/hooks/test_mysql.py | 32 +++++++++++++++------- tests/unit/test_execute.py | 35 ++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 26 deletions(-) diff --git a/NEWS b/NEWS index 7d02c0e2..472eef44 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 1.7.10.dev0 + * #396: When a database command errors, display and log the error message instead of swallowing it. * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" option in borgmatic's location configuration. * #576: Add support for "file://" paths within "repositories" option. diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 46dd04f3..0afa5cca 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -43,6 +43,23 @@ def output_buffer_for_process(process, exclude_stdouts): return process.stderr if process.stdout in exclude_stdouts else process.stdout +def append_last_lines(last_lines, captured_output, line, output_log_level): + ''' + Given a rolling list of last lines, a list of captured output, a line to append, and an output + log level, append the line to the last lines and (if necessary) the captured output. Then log + the line at the requested output log level. + ''' + last_lines.append(line) + + if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT: + last_lines.pop(0) + + if output_log_level is None: + captured_output.append(line) + else: + logger.log(output_log_level, line) + + def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): ''' Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each @@ -98,15 +115,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): # Keep the last few lines of output in case the process errors, and we need the output for # the exception below. - last_lines = buffer_last_lines[ready_buffer] - last_lines.append(line) - if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT: - last_lines.pop(0) - - if output_log_level is None: - captured_outputs[ready_process].append(line) - else: - logger.log(output_log_level, line) + append_last_lines( + buffer_last_lines[ready_buffer], + captured_outputs[ready_process], + line, + output_log_level, + ) if not still_running: break @@ -125,8 +139,18 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): # If an error occurs, include its output in the raised exception so that we don't # inadvertently hide error output. output_buffer = output_buffer_for_process(process, exclude_stdouts) - last_lines = buffer_last_lines[output_buffer] if output_buffer else [] + + # Collect any straggling output lines that came in since we last gathered output. + while output_buffer: # pragma: no cover + line = output_buffer.readline().rstrip().decode() + if not line: + break + + append_last_lines( + last_lines, captured_outputs[process], line, output_log_level=logging.ERROR + ) + if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT: last_lines.insert(0, '...') diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 0bf97745..aeeeee5b 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -88,9 +88,7 @@ def execute_dump_command( + (('--user', database['username']) if 'username' in database else ()) + ('--databases',) + database_names - # Use shell redirection rather than execute_command(output_file=open(...)) to prevent - # the open() call on a named pipe from hanging the main borgmatic process. - + ('>', dump_filename) + + ('--result-file', dump_filename) ) logger.debug( @@ -102,7 +100,7 @@ def execute_dump_command( dump.create_named_pipe_for_dump(dump_filename) return execute_command( - dump_command, shell=True, extra_environment=extra_environment, run_to_completion=False, + dump_command, extra_environment=extra_environment, run_to_completion=False, ) diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 2a9b61d7..09e9093a 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -239,7 +239,6 @@ def test_log_outputs_does_not_error_when_one_process_exits(): def test_log_outputs_truncates_long_error_output(): - flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0 flexmock(module.logger).should_receive('log') flexmock(module).should_receive('command_for_process').and_return('grep') @@ -253,7 +252,7 @@ def test_log_outputs_truncates_long_error_output(): flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout) with pytest.raises(subprocess.CalledProcessError) as error: - module.log_outputs( + flexmock(module, ERROR_OUTPUT_MAX_LINE_COUNT=0).log_outputs( (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg' ) diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 1e8df691..137a88a5 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -149,8 +149,7 @@ def test_execute_dump_command_runs_mysqldump(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--add-drop-database', '--databases', 'foo', '>', 'dump',), - shell=True, + ('mysqldump', '--add-drop-database', '--databases', 'foo', '--result-file', 'dump',), extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -176,8 +175,7 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--databases', 'foo', '>', 'dump',), - shell=True, + ('mysqldump', '--databases', 'foo', '--result-file', 'dump',), extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -214,10 +212,9 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): 'tcp', '--databases', 'foo', - '>', + '--result-file', 'dump', ), - shell=True, extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -243,8 +240,16 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--add-drop-database', '--user', 'root', '--databases', 'foo', '>', 'dump',), - shell=True, + ( + 'mysqldump', + '--add-drop-database', + '--user', + 'root', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment={'MYSQL_PWD': 'trustsome1'}, run_to_completion=False, ).and_return(process).once() @@ -270,8 +275,15 @@ def test_execute_dump_command_runs_mysqldump_with_options(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--stuff=such', '--add-drop-database', '--databases', 'foo', '>', 'dump',), - shell=True, + ( + 'mysqldump', + '--stuff=such', + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment=None, run_to_completion=False, ).and_return(process).once() diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index badcb8fd..14924078 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -65,6 +65,41 @@ def test_output_buffer_for_process_returns_stdout_when_not_excluded(): ) +def test_append_last_lines_under_max_line_count_appends(): + last_lines = ['last'] + flexmock(module.logger).should_receive('log').once() + + module.append_last_lines( + last_lines, captured_output=flexmock(), line='line', output_log_level=flexmock() + ) + + assert last_lines == ['last', 'line'] + + +def test_append_last_lines_over_max_line_count_trims_and_appends(): + original_last_lines = [str(number) for number in range(0, module.ERROR_OUTPUT_MAX_LINE_COUNT)] + last_lines = list(original_last_lines) + flexmock(module.logger).should_receive('log').once() + + module.append_last_lines( + last_lines, captured_output=flexmock(), line='line', output_log_level=flexmock() + ) + + assert last_lines == original_last_lines[1:] + ['line'] + + +def test_append_last_lines_with_output_log_level_none_appends_captured_output(): + last_lines = ['last'] + captured_output = ['captured'] + flexmock(module.logger).should_receive('log').never() + + module.append_last_lines( + last_lines, captured_output=captured_output, line='line', output_log_level=None + ) + + assert captured_output == ['captured', 'line'] + + def test_execute_command_calls_full_command(): full_command = ['foo', 'bar'] flexmock(module.os, environ={'a': 'b'}) From 4fa4fccab7b186522648a2d78c9098937e86d8b6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 27 Mar 2023 23:24:17 +0530 Subject: [PATCH 015/344] Use make_flags_from_arguments on mount; Pending test fixes Signed-off-by: Chirag Aggarwal --- borgmatic/borg/mount.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index d987ca4e..27975348 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -9,10 +9,6 @@ logger = logging.getLogger(__name__) def mount_archive( repository, archive, - mount_point, - paths, - foreground, - options, mount_arguments, storage_config, local_borg_version, @@ -35,14 +31,13 @@ def mount_archive( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) - + (('--foreground',) if foreground else ()) - + (flags.make_flags('first', mount_arguments.first) if mount_arguments.first else ()) - + (flags.make_flags('last', mount_arguments.last) if mount_arguments.last else ()) - + (flags.make_flags('newest', mount_arguments.newest) if mount_arguments.newest else ()) - + (flags.make_flags('oldest', mount_arguments.oldest) if mount_arguments.oldest else ()) - + (flags.make_flags('older', mount_arguments.older) if mount_arguments.older else ()) - + (flags.make_flags('newer', mount_arguments.newer) if mount_arguments.newer else ()) - + (('-o', options) if options else ()) + + + flags.make_flags_from_arguments( + mount_arguments, + excludes=('repository', 'archive', 'mount_point', 'path', 'options'), + ) + + + (('-o', mount_arguments.options) if mount_arguments.options else ()) + ( ( flags.make_repository_flags(repository, local_borg_version) @@ -59,14 +54,14 @@ def mount_archive( else flags.make_repository_flags(repository, local_borg_version) ) ) - + (mount_point,) - + (tuple(paths) if paths else ()) + + (mount_arguments.mount_point,) + + (tuple(mount_arguments.paths) if mount_arguments.paths else ()) ) borg_environment = environment.make_environment(storage_config) # Don't capture the output when foreground mode is used so that ctrl-C can work properly. - if foreground: + if mount_arguments.foreground: execute_command( full_command, output_file=DO_NOT_CAPTURE, From 61c7b8f13cf6a3c905cd82cdc7dac6b5cb8dd421 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 15:54:55 -0700 Subject: [PATCH 016/344] Add optional repository labels so you can select a repository via "--repository yourlabel" at the command-line (#635). --- NEWS | 3 +++ README.md | 7 ++++--- docs/how-to/make-backups-redundant.md | 5 ++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index 472eef44..ed2bafc1 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,9 @@ * #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option in borgmatic's storage configuration. * #623: Fix confusing message when an error occurs running actions for a configuration file. + * #635: Add optional repository labels so you can select a repository via "--repository yourlabel" + at the command-line. See the configuration reference for more information: + https://torsion.org/borgmatic/docs/reference/configuration/ * #649: Add documentation on backing up a database running in a container: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers * #655: Fix error when databases are configured and a source directory doesn't exist. diff --git a/README.md b/README.md index 27fc6cd2..eb827ae9 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,10 @@ location: # Paths of local or remote repositories to backup to. repositories: - - ssh://1234@usw-s001.rsync.net/./backups.borg - - ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo - - /var/lib/backups/local.borg + - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo + label: borgbase + - path: /var/lib/backups/local.borg + label: local retention: # Retention policy for how many backups to keep. diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md index f77780fc..71510fc4 100644 --- a/docs/how-to/make-backups-redundant.md +++ b/docs/how-to/make-backups-redundant.md @@ -20,9 +20,8 @@ location: # Paths of local or remote repositories to backup to. repositories: - - ssh://1234@usw-s001.rsync.net/./backups.borg - - ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo - - /var/lib/backups/local.borg + - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo + - path: /var/lib/backups/local.borg ``` When you run borgmatic with this configuration, it invokes Borg once for each From c5ffb76dfa4344ad582e093a25a4488381adb396 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 15:56:49 -0700 Subject: [PATCH 017/344] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index ed2bafc1..9bc9ebf6 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.10.dev0 +1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" option in borgmatic's location configuration. diff --git a/setup.py b/setup.py index 5ea3c2e4..c3682115 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.10.dev0' +VERSION = '1.7.10' setup( From 0e3da7be6341f9b53b74a7bc0f4cac596af9a574 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 16:15:24 -0700 Subject: [PATCH 018/344] Fix repository schema description. --- borgmatic/config/schema.yaml | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 9115eab1..f0dc3b32 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -46,29 +46,20 @@ properties: properties: path: type: string - description: | - Path to local or remote repository - (required). Tildes are expanded. - Multiple repositories are backed up - to in sequence. Borg placeholders - can be used. See the output of - "borg help placeholders" for - details. See ssh_command for SSH - options like identity file or port. - If systemd service is used, then - add local repository paths in the - systemd service file to the - ReadWritePaths list. example: ssh://user@backupserver/./{fqdn} label: type: string - description: | - Optional label for the repository. This - can be used with the --repository flag - to select a repository to backup to. - If not specified, the repository path is - used as the label. example: backupserver + description: | + A required list of local or remote repositories with paths + and optional labels (which can be used with the --repository + flag to select a repository). Tildes are expanded. Multiple + repositories are backed up to in sequence. Borg placeholders + can be used. See the output of "borg help placeholders" for + details. See ssh_command for SSH options like identity file + or port. If systemd service is used, then add local + repository paths in the systemd service file to the + ReadWritePaths list. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver From 489ae080e525c5b9c735124d269c3de6f6f43c58 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 21:49:31 -0700 Subject: [PATCH 019/344] Update docs with a few more "path:" repositories references (#635). --- ...p-to-a-removable-drive-or-an-intermittent-server.md | 10 ++++++++-- docs/how-to/make-backups-redundant.md | 3 +++ docs/how-to/upgrade.md | 7 +++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index 0f578551..098b03f6 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -49,9 +49,12 @@ location: - /home repositories: - - /mnt/removable/backup.borg + - path: /mnt/removable/backup.borg ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + Then, write a `before_backup` hook in that same configuration file that uses the external `findmnt` utility to see whether the drive is mounted before proceeding. @@ -79,13 +82,16 @@ location: - /home repositories: - - ssh://me@buddys-server.org/./backup.borg + - path: ssh://me@buddys-server.org/./backup.borg hooks: before_backup: - ping -q -c 1 buddys-server.org > /dev/null || exit 75 ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + Or to only run backups if the battery level is high enough: ```yaml diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md index 71510fc4..a7e21028 100644 --- a/docs/how-to/make-backups-redundant.md +++ b/docs/how-to/make-backups-redundant.md @@ -24,6 +24,9 @@ location: - path: /var/lib/backups/local.borg ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + When you run borgmatic with this configuration, it invokes Borg once for each configured repository in sequence. (So, not in parallel.) That means—in each repository—borgmatic creates a single new backup archive containing all of diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index 296b3f88..69b5f5b8 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -145,15 +145,18 @@ like this: ```yaml location: repositories: - - original.borg + - path: original.borg ``` +Prior to version 1.7.10 Omit +the `path:` portion of the `repositories` list. + Change it to a new (not yet created) repository path: ```yaml location: repositories: - - upgraded.borg + - path: upgraded.borg ``` Then, run the `rcreate` action (formerly `init`) to create that new Borg 2 From 7a8e0e89ddcbfcc36722df102bd29c3c5ff49b7e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 21:54:01 -0700 Subject: [PATCH 020/344] Mention prior versions of borgmatic in repositories schema. --- borgmatic/config/schema.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index f0dc3b32..3e94b8d2 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -59,7 +59,8 @@ properties: details. See ssh_command for SSH options like identity file or port. If systemd service is used, then add local repository paths in the systemd service file to the - ReadWritePaths list. + ReadWritePaths list. Prior to borgmatic 1.7.10, repositories + was just a list of plain path strings. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver From 1bcdebd1cc6731142cbbcee04d551fbb09480c8a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 23:16:44 -0700 Subject: [PATCH 021/344] Fix multiple repositories example. --- docs/how-to/make-backups-redundant.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md index a7e21028..2a4b8121 100644 --- a/docs/how-to/make-backups-redundant.md +++ b/docs/how-to/make-backups-redundant.md @@ -34,9 +34,8 @@ your source directories. Here's a way of visualizing what borgmatic does with the above configuration: -1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg` -2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo` -3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg` +1. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo` +2. Backup `/home` and `/etc` to `/var/lib/backups/local.borg` This gives you redundancy of your data across repositories and even potentially across providers. From dcefded0fa5fd88ceddf48e62826d3ef99351c0d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 23:21:14 -0700 Subject: [PATCH 022/344] Document that most command-line flags are not config-file-able (#461). --- docs/reference/command-line.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/reference/command-line.md b/docs/reference/command-line.md index 8997cd93..d1b67dc5 100644 --- a/docs/reference/command-line.md +++ b/docs/reference/command-line.md @@ -7,8 +7,10 @@ eleventyNavigation: --- ## borgmatic options -Here are all of the available borgmatic command-line options. This includes the separate options for -each action sub-command: +Here are all of the available borgmatic command-line options, including the +separate options for each action sub-command. Note that most of the +flags listed here do have equivalents in borgmatic's [configuration +file](https://torsion.org/borgmatic/docs/reference/configuration/). ``` {% include borgmatic/command-line.txt %} From 67a349ae4418718e73cf0b90753ba868e18f7735 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 23:28:36 -0700 Subject: [PATCH 023/344] I had one job... (#461). --- docs/reference/command-line.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/command-line.md b/docs/reference/command-line.md index d1b67dc5..9cfdd7ee 100644 --- a/docs/reference/command-line.md +++ b/docs/reference/command-line.md @@ -9,7 +9,7 @@ eleventyNavigation: Here are all of the available borgmatic command-line options, including the separate options for each action sub-command. Note that most of the -flags listed here do have equivalents in borgmatic's [configuration +flags listed here do not have equivalents in borgmatic's [configuration file](https://torsion.org/borgmatic/docs/reference/configuration/). ``` From d96f2239c1a50c9376b50fbde570a9b2074da9f0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 23:43:39 -0700 Subject: [PATCH 024/344] Update OpenBSD borgmatic link. --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 52962c34..2ab1e8d5 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -90,7 +90,7 @@ installing borgmatic: * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) - * [OpenBSD](http://ports.su/sysutils/borgmatic) + * [OpenBSD](https://openports.pl/path/sysutils/borgmatic) * [openSUSE](https://software.opensuse.org/package/borgmatic) * [macOS (via Homebrew)](https://formulae.brew.sh/formula/borgmatic) * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/) From edd79ed86cca097216741363151deebe35839e25 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 28 Mar 2023 18:10:42 +0530 Subject: [PATCH 025/344] removed individual action parameters, and used make_flags_from_arguments Signed-off-by: Chirag Aggarwal --- borgmatic/actions/mount.py | 4 ---- borgmatic/actions/prune.py | 2 -- borgmatic/borg/prune.py | 19 +++++++++---------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index 2b661ec2..b1a1132f 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -31,10 +31,6 @@ def run_mount( local_path, remote_path, ), - mount_arguments.mount_point, - mount_arguments.paths, - mount_arguments.foreground, - mount_arguments.options, mount_arguments, storage, local_borg_version, diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index a89b717f..09666ee8 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -47,8 +47,6 @@ def run_prune( prune_arguments, local_path=local_path, remote_path=remote_path, - stats=prune_arguments.stats, - list_archives=prune_arguments.list_archives, ) borgmatic.hooks.command.execute_hook( hooks.get('after_prune'), diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index aa81a7f9..b8b1d6b2 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -46,8 +46,6 @@ def prune_archives( prune_arguments, local_path='borg', remote_path=None, - stats=False, - list_archives=False, ): ''' Given dry-run flag, a local or remote repository path, a storage config dict, and a @@ -69,20 +67,21 @@ def prune_archives( + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) - + (('--stats',) if stats and not dry_run else ()) + + (('--stats',) if prune_arguments.stats and not dry_run else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) - + (('--list',) if list_archives else ()) - + (flags.make_flags('newest', prune_arguments.newest) if prune_arguments.newest else ()) - + (flags.make_flags('oldest', prune_arguments.oldest) if prune_arguments.oldest else ()) - + (flags.make_flags('older', prune_arguments.older) if prune_arguments.older else ()) - + (flags.make_flags('newer', prune_arguments.newer) if prune_arguments.newer else ()) + + + flags.make_flags_from_arguments( + prune_arguments, + excludes=('repository', 'stats'), + ) + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) - + (('--dry-run',) if dry_run else ()) + + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository, local_borg_version) ) - if stats or list_archives: + if prune_arguments.stats or prune_arguments.list_archives: output_log_level = logging.ANSWER else: output_log_level = logging.INFO From 98c6aa644379dee2c1d89ee30d2f2c9588c978fe Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 28 Mar 2023 18:15:49 +0530 Subject: [PATCH 026/344] Use Square brackets to denote version specific flag Signed-off-by: Chirag Aggarwal --- borgmatic/commands/arguments.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index b7ee94aa..db743dc4 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -246,7 +246,7 @@ def make_parsers(): '--source-repository', '--other-repo', metavar='KEY_REPOSITORY', - help='Path to an existing Borg repository whose key material should be reused (Borg 2.x+ only)', + help='Path to an existing Borg repository whose key material should be reused [Borg 2.x+ only]', ) rcreate_group.add_argument( '--repository', @@ -255,7 +255,7 @@ def make_parsers(): rcreate_group.add_argument( '--copy-crypt-key', action='store_true', - help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key (Borg 2.x+ only)', + help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key [Borg 2.x+ only]', ) rcreate_group.add_argument( '--append-only', action='store_true', help='Create an append-only repository', @@ -275,8 +275,8 @@ def make_parsers(): transfer_parser = subparsers.add_parser( 'transfer', aliases=SUBPARSER_ALIASES['transfer'], - help='Transfer archives from one repository to another, optionally upgrading the transferred data (Borg 2.0+ only)', - description='Transfer archives from one repository to another, optionally upgrading the transferred data (Borg 2.0+ only)', + help='Transfer archives from one repository to another, optionally upgrading the transferred data [Borg 2.0+ only]', + description='Transfer archives from one repository to another, optionally upgrading the transferred data [Borg 2.0+ only]', add_help=False, ) transfer_group = transfer_parser.add_argument_group('transfer arguments') @@ -325,7 +325,7 @@ def make_parsers(): '--oldest', metavar='TIMESPAN', help='Transfer archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' ) transfer_group.add_argument( - '--newest', metavar='TIMESPAN', help='Transfer archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', metavar='TIMESPAN', help='Transfer archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' ) transfer_group.add_argument( '--older', metavar='TIMESPAN', help='Transfer archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' @@ -363,7 +363,7 @@ def make_parsers(): '--oldest', metavar='TIMESPAN', help='Prune archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' ) prune_group.add_argument( - '--newest', metavar='TIMESPAN', help='Prune archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', metavar='TIMESPAN', help='Prune archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' ) prune_group.add_argument( '--older', metavar='TIMESPAN', help='Prune archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' @@ -376,8 +376,8 @@ def make_parsers(): compact_parser = subparsers.add_parser( 'compact', aliases=SUBPARSER_ALIASES['compact'], - help='Compact segments to free space (Borg 1.2+, borgmatic 1.5.23+ only)', - description='Compact segments to free space (Borg 1.2+, borgmatic 1.5.23+ only)', + help='Compact segments to free space [Borg 1.2+, borgmatic 1.5.23+ only]', + description='Compact segments to free space [Borg 1.2+, borgmatic 1.5.23+ only]', add_help=False, ) compact_group = compact_parser.add_argument_group('compact arguments') @@ -397,7 +397,7 @@ def make_parsers(): dest='cleanup_commits', default=False, action='store_true', - help='Cleanup commit-only 17-byte segment files left behind by Borg 1.1 (flag in Borg 1.2 only)', + help='Cleanup commit-only 17-byte segment files left behind by Borg 1.1 [flag in Borg 1.2 only]', ) compact_group.add_argument( '--threshold', @@ -623,7 +623,7 @@ def make_parsers(): '--oldest', metavar='TIMESPAN', help='Mount archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' ) mount_group.add_argument( - '--newest', metavar='TIMESPAN', help='Mount archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', metavar='TIMESPAN', help='Mount archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' ) mount_group.add_argument( '--older', metavar='TIMESPAN', help='Mount archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' @@ -718,7 +718,7 @@ def make_parsers(): '--oldest', metavar='TIMESPAN', help='List archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' ) rlist_group.add_argument( - '--newest', metavar='TIMESPAN', help='List archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', metavar='TIMESPAN', help='List archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' ) rlist_group.add_argument( '--older', metavar='TIMESPAN', help='List archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' @@ -859,7 +859,7 @@ def make_parsers(): '--oldest', metavar='TIMESPAN', help='Show info for archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' ) info_group.add_argument( - '--newest', metavar='TIMESPAN', help='Show info for archives within a time range that ends at newest archive\'s timestamp and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', metavar='TIMESPAN', help='Show info for archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' ) info_group.add_argument( '--older', metavar='TIMESPAN', help='Show info for archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' From 2d08a63e603d53ec2a949405cdccbbf809bcad37 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 28 Mar 2023 22:14:50 +0530 Subject: [PATCH 027/344] fix: make check repositories work with dict and str repositories --- borgmatic/config/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index abcfe3d2..96025b3a 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -69,7 +69,7 @@ def apply_logical_validation(config_filename, parsed_configuration): location_repositories = parsed_configuration.get('location', {}).get('repositories') check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) for repository in check_repositories: - if repository not in location_repositories: + if not any(repositories_match(repository, config_repository) for config_repository in location_repositories): raise Validation_error( config_filename, ( From ce22d2d30252a23ff8aea0c9afb0491d618aca75 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 28 Mar 2023 22:29:21 +0530 Subject: [PATCH 028/344] reformat --- borgmatic/config/validate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 96025b3a..fcf29d38 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -69,7 +69,10 @@ def apply_logical_validation(config_filename, parsed_configuration): location_repositories = parsed_configuration.get('location', {}).get('repositories') check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) for repository in check_repositories: - if not any(repositories_match(repository, config_repository) for config_repository in location_repositories): + if not any( + repositories_match(repository, config_repository) + for config_repository in location_repositories + ): raise Validation_error( config_filename, ( From 08e358e27f09b84f08e932973ddf666356c889d4 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 28 Mar 2023 22:51:35 +0530 Subject: [PATCH 029/344] add and update tests --- tests/unit/config/test_validate.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 327a2b44..d5606bde 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -51,17 +51,33 @@ def test_apply_locical_validation_raises_if_unknown_repository_in_check_reposito ) -def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_repositories(): +def test_apply_locical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { - 'location': {'repositories': ['repo.borg', 'other.borg']}, + 'location': {'repositories': [{'path': 'repo.borg'}, {'path': 'other.borg'}]}, 'retention': {'keep_secondly': 1000}, 'consistency': {'check_repositories': ['repo.borg']}, }, ) +def test_apply_locical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): + module.apply_logical_validation( + 'config.yaml', + { + 'location': { + 'repositories': [ + {'path': 'repo.borg', 'label': 'my_repo'}, + {'path': 'other.borg', 'label': 'other_repo'}, + ] + }, + 'retention': {'keep_secondly': 1000}, + 'consistency': {'check_repositories': ['my_repo']}, + }, + ) + + def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present(): module.apply_logical_validation( 'config.yaml', From 59fe01b56d18e92ee49f6b8417c8f867a3f904df Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 11:09:25 -0700 Subject: [PATCH 030/344] Update script comment. --- scripts/run-end-to-end-dev-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run-end-to-end-dev-tests b/scripts/run-end-to-end-dev-tests index 3ee37acc..575de00d 100755 --- a/scripts/run-end-to-end-dev-tests +++ b/scripts/run-end-to-end-dev-tests @@ -1,7 +1,7 @@ #!/bin/sh -# This script is for running all tests, including end-to-end tests, on a developer machine. It sets -# up database containers to run tests against, runs the tests, and then tears down the containers. +# This script is for running end-to-end tests on a developer machine. It sets up database containers +# to run tests against, runs the tests, and then tears down the containers. # # Run this script from the root directory of the borgmatic source. # From 3512191f3e8f3c4ac537746dc35f34631e7a19ba Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 11:45:55 -0700 Subject: [PATCH 031/344] Add check_repositories regression fix to NEWS (#662). --- NEWS | 3 +++ setup.py | 2 +- tests/unit/config/test_validate.py | 6 +++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 9bc9ebf6..7f2353d6 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.7.11.dev0 + * #662: Fix regression in which "check_repositories" option failed to match repositories. + 1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" diff --git a/setup.py b/setup.py index c3682115..bc4e4b75 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.10' +VERSION = '1.7.11.dev0' setup( diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index d5606bde..11f3127c 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -37,7 +37,7 @@ def test_validation_error_string_contains_errors(): assert 'uh oh' in result -def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories(): +def test_apply_logical_validation_raises_if_unknown_repository_in_check_repositories(): flexmock(module).format_json_error = lambda error: error.message with pytest.raises(module.Validation_error): @@ -51,7 +51,7 @@ def test_apply_locical_validation_raises_if_unknown_repository_in_check_reposito ) -def test_apply_locical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): +def test_apply_logical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { @@ -62,7 +62,7 @@ def test_apply_locical_validation_does_not_raise_if_known_repository_path_in_che ) -def test_apply_locical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): +def test_apply_logical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { From f709125110eb4c58204ed19b3ee591497523d36d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 12:02:07 -0700 Subject: [PATCH 032/344] Error out if run-full-tests is run not inside a test container. --- .drone.yml | 2 ++ scripts/run-full-tests | 9 ++++++++- tests/end-to-end/docker-compose.yaml | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index cf4c358a..354cff95 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,6 +24,8 @@ clone: steps: - name: build image: alpine:3.13 + environment: + TEST_CONTAINER: true pull: always commands: - scripts/run-full-tests diff --git a/scripts/run-full-tests b/scripts/run-full-tests index cbd824ce..bf26c212 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -8,7 +8,14 @@ # For more information, see: # https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/ -set -ex +set -e + +if [ -z "$TEST_CONTAINER" ] ; then + echo "This script is designed to work inside a test container and is not intended to" + echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute" + echo "scripts/run-end-to-end-dev-tests instead." + exit 1 +fi apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \ py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 090cf12c..884edba0 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -17,6 +17,8 @@ services: MONGO_INITDB_ROOT_PASSWORD: test tests: image: alpine:3.13 + environment: + TEST_CONTAINER: true volumes: - "../..:/app:ro" tmpfs: From aaf3462d1704a39149708180ab9bf2db9b145244 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 12:03:12 -0700 Subject: [PATCH 033/344] Fix Drone intentation. --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 354cff95..9a7630ac 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,8 +24,8 @@ clone: steps: - name: build image: alpine:3.13 - environment: - TEST_CONTAINER: true + environment: + TEST_CONTAINER: true pull: always commands: - scripts/run-full-tests From 010b82d6d8302616c8fa34973d795afa992f6315 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 12:45:39 -0700 Subject: [PATCH 034/344] Remove unnecessary cd in dev documentation. --- docs/how-to/develop-on-borgmatic.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index d4315457..796c7367 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -25,7 +25,7 @@ so that you can run borgmatic commands while you're hacking on them to make sure your changes work. ```bash -cd borgmatic/ +cd borgmatic pip3 install --user --editable . ``` @@ -51,7 +51,6 @@ pip3 install --user tox Finally, to actually run tests, run: ```bash -cd borgmatic tox ``` From fc2c181b740b516fab6fb9e9e0907aea5707cc82 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 15:31:37 -0700 Subject: [PATCH 035/344] Add missing Docker Compose depends. --- tests/end-to-end/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 884edba0..80f12e9a 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -30,3 +30,4 @@ services: depends_on: - postgresql - mysql + - mongodb From b27e625a7772860fb5ab2042c87bcc74b98556d8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 15:44:38 -0700 Subject: [PATCH 036/344] Update schema comment for check_repositories to mention labels (#635). --- borgmatic/config/schema.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3e94b8d2..469b1f52 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -538,12 +538,12 @@ properties: items: type: string description: | - Paths to a subset of the repositories in the location - section on which to run consistency checks. Handy in case - some of your repositories are very large, and so running - consistency checks on them would take too long. Defaults to - running consistency checks on all repositories configured in - the location section. + Paths or labels for a subset of the repositories in the + location section on which to run consistency checks. Handy + in case some of your repositories are very large, and so + running consistency checks on them would take too long. + Defaults to running consistency checks on all repositories + configured in the location section. example: - user@backupserver:sourcehostname.borg check_last: From 5f595f7ac3fbfc8c19694baaa8b08e613b60a98b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 30 Mar 2023 23:21:20 -0700 Subject: [PATCH 037/344] Fix regression in which the "transfer" action produced a traceback (#663). --- NEWS | 1 + borgmatic/actions/transfer.py | 4 ++-- tests/unit/actions/test_transfer.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 7f2353d6..8ffbbde3 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.7.11.dev0 * #662: Fix regression in which "check_repositories" option failed to match repositories. + * #663: Fix regression in which the "transfer" action produced a traceback. 1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 628f2735..8089fd4e 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,10 +17,10 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' - logger.info(f'{repository}: Transferring archives to repository') + logger.info(f'{repository["path"]}: Transferring archives to repository') borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, transfer_arguments, diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py index cc9f1386..58d8a160 100644 --- a/tests/unit/actions/test_transfer.py +++ b/tests/unit/actions/test_transfer.py @@ -10,7 +10,7 @@ def test_run_transfer_does_not_raise(): global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_transfer( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, transfer_arguments=transfer_arguments, From 3f78ac4085f689839d9f84d5328cd34b53333e29 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 31 Mar 2023 15:21:08 -0700 Subject: [PATCH 038/344] Automatically use the "archive_name_format" option to filter which archives get used for borgmatic actions that operate on multiple archives (#479). --- NEWS | 5 ++ borgmatic/borg/check.py | 29 +++--- borgmatic/borg/flags.py | 18 ++++ borgmatic/borg/info.py | 6 +- borgmatic/borg/prune.py | 22 ++--- borgmatic/borg/rlist.py | 6 +- borgmatic/borg/transfer.py | 13 ++- borgmatic/config/schema.yaml | 29 +++--- docs/how-to/make-per-application-backups.md | 90 +++++++++++++++---- tests/unit/borg/test_check.py | 99 +++++++++++++-------- tests/unit/borg/test_flags.py | 33 +++++++ tests/unit/borg/test_info.py | 91 ++++++++++++++++++- tests/unit/borg/test_prune.py | 61 +++++++------ tests/unit/borg/test_rlist.py | 69 ++++++++++++++ tests/unit/borg/test_transfer.py | 45 +++++++++- 15 files changed, 488 insertions(+), 128 deletions(-) diff --git a/NEWS b/NEWS index 8ffbbde3..f0518876 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,9 @@ 1.7.11.dev0 + * #479: Automatically use the "archive_name_format" option to filter which archives get used for + borgmatic actions that operate on multiple archives. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming + * #479: The "prefix" options have been deprecated in favor of the new "archive_name_format" + auto-matching behavior (see above). * #662: Fix regression in which "check_repositories" option failed to match repositories. * #663: Fix regression in which the "transfer" action produced a traceback. diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 993a3c0a..4630782a 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -12,7 +12,6 @@ DEFAULT_CHECKS = ( {'name': 'repository', 'frequency': '1 month'}, {'name': 'archives', 'frequency': '1 month'}, ) -DEFAULT_PREFIX = '{hostname}-' # noqa: FS003 logger = logging.getLogger(__name__) @@ -146,9 +145,10 @@ def filter_checks_on_frequency( return tuple(filtered_checks) -def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): +def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None): ''' - Given the local Borg version and a parsed sequence of checks, transform the checks into tuple of + Given the local Borg version, a storge configuration dict, a parsed sequence of checks, the + check last value, and a consistency check prefix, transform the checks into tuple of command-line flags. For example, given parsed checks of: @@ -174,10 +174,19 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): if 'archives' in checks: last_flags = ('--last', str(check_last)) if check_last else () - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): - match_archives_flags = ('--match-archives', f'sh:{prefix}*') if prefix else () - else: - match_archives_flags = ('--glob-archives', f'{prefix}*') if prefix else () + match_archives_flags = ( + ( + ('--match-archives', f'sh:{prefix}*') + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) + else ('--glob-archives', f'{prefix}*') + ) + if prefix + else ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) + ) else: last_flags = () match_archives_flags = () @@ -291,7 +300,7 @@ def check_archives( extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') if set(checks).intersection({'repository', 'archives', 'data'}): - lock_wait = storage_config.get('lock_wait', None) + lock_wait = storage_config.get('lock_wait') verbosity_flags = () if logger.isEnabledFor(logging.INFO): @@ -299,12 +308,12 @@ def check_archives( if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') - prefix = consistency_config.get('prefix', DEFAULT_PREFIX) + prefix = consistency_config.get('prefix') full_command = ( (local_path, 'check') + (('--repair',) if repair else ()) - + make_check_flags(local_borg_version, checks, check_last, prefix) + + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix) + (('--remote-path', remote_path) if remote_path else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 5dcebf50..1354d658 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -1,4 +1,5 @@ import itertools +import re from borgmatic.borg import feature @@ -56,3 +57,20 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version): if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) else (f'{repository_path}::{archive}',) ) + + +def make_match_archives_flags(archive_name_format, local_borg_version): + ''' + Return the match archives flags that would match archives created with the given archive name + format (if any). This is done by replacing certain archive name format placeholders for + ephemeral data (like "{now}") with globs. + ''' + if not archive_name_format: + return () + + match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format) + + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): + return ('--match-archives', f'sh:{match_archives}') + else: + return ('--glob-archives', f'{match_archives}') diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index 6142104f..fa59165c 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -44,7 +44,11 @@ def display_archives_info( else flags.make_flags('glob-archives', f'{info_arguments.prefix}*') ) if info_arguments.prefix - else () + else ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) ) + flags.make_flags_from_arguments( info_arguments, excludes=('repository', 'archive', 'prefix') diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index d21ceee3..08cb0c17 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -7,10 +7,10 @@ from borgmatic.execute import execute_command logger = logging.getLogger(__name__) -def make_prune_flags(retention_config, local_borg_version): +def make_prune_flags(storage_config, retention_config, local_borg_version): ''' - Given a retention config dict mapping from option name to value, tranform it into an iterable of - command-line name-value flag pairs. + Given a retention config dict mapping from option name to value, tranform it into an sequence of + command-line flags. For example, given a retention config of: @@ -24,7 +24,7 @@ def make_prune_flags(retention_config, local_borg_version): ) ''' config = retention_config.copy() - prefix = config.pop('prefix', '{hostname}-') # noqa: FS003 + prefix = config.pop('prefix', None) if prefix: if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): @@ -32,10 +32,16 @@ def make_prune_flags(retention_config, local_borg_version): else: config['glob_archives'] = f'{prefix}*' - return ( + flag_pairs = ( ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items() ) + return tuple( + element for pair in flag_pairs for element in pair + ) + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + def prune_archives( dry_run, @@ -60,11 +66,7 @@ def prune_archives( full_command = ( (local_path, 'prune') - + tuple( - element - for pair in make_prune_flags(retention_config, local_borg_version) - for element in pair - ) + + make_prune_flags(storage_config, retention_config, local_borg_version) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 8625363b..f3935a16 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -94,7 +94,11 @@ def make_rlist_command( else flags.make_flags('glob-archives', f'{rlist_arguments.prefix}*') ) if rlist_arguments.prefix - else () + else ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) ) + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 29e205c7..a350eff7 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -34,9 +34,16 @@ def transfer_archives( 'match-archives', transfer_arguments.match_archives or transfer_arguments.archive ) ) - + flags.make_flags_from_arguments( - transfer_arguments, - excludes=('repository', 'source_repository', 'archive', 'match_archives'), + + ( + flags.make_flags_from_arguments( + transfer_arguments, + excludes=('repository', 'source_repository', 'archive', 'match_archives'), + ) + or ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) ) + flags.make_repository_flags(repository_path, local_borg_version) + flags.make_flags('other-repo', transfer_arguments.source_repository) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 469b1f52..d4374525 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -378,11 +378,9 @@ properties: description: | Name of the archive. Borg placeholders can be used. See the output of "borg help placeholders" for details. Defaults to - "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this - option, consider also specifying a prefix in the retention - and consistency sections to avoid accidental - pruning/checking of archives with different archive name - formats. + "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". When running + actions like rlist, info, or check, borgmatic automatically + tries to match only archives created with this name format. example: "{hostname}-documents-{now}" relocated_repo_access_is_ok: type: boolean @@ -477,10 +475,12 @@ properties: prefix: type: string description: | - When pruning, only consider archive names starting with this - prefix. Borg placeholders can be used. See the output of - "borg help placeholders" for details. Defaults to - "{hostname}-". Use an empty value to disable the default. + Deprecated. When pruning, only consider archive names + starting with this prefix. Borg placeholders can be used. + See the output of "borg help placeholders" for details. + If a prefix is not specified, borgmatic defaults to + matching archives based on the archive_name_format (see + above). example: sourcehostname consistency: type: object @@ -556,11 +556,12 @@ properties: prefix: type: string description: | - When performing the "archives" check, only consider archive - names starting with this prefix. Borg placeholders can be - used. See the output of "borg help placeholders" for - details. Defaults to "{hostname}-". Use an empty value to - disable the default. + Deprecated. When performing the "archives" check, only + consider archive names starting with this prefix. Borg + placeholders can be used. See the output of "borg help + placeholders" for details. If a prefix is not specified, + borgmatic defaults to matching archives based on the + archive_name_format (see above). example: sourcehostname output: type: object diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index e5ba037f..6e5d999a 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -54,6 +54,72 @@ choice](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot), each entry using borgmatic's `--config` flag instead of relying on `/etc/borgmatic.d`. + +## Archive naming + +If you've got multiple borgmatic configuration files, you might want to create +archives with different naming schemes for each one. This is especially handy +if each configuration file is backing up to the same Borg repository but you +still want to be able to distinguish backup archives for one application from +another. + +borgmatic supports this use case with an `archive_name_format` option. The +idea is that you define a string format containing a number of [Borg +placeholders](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-placeholders), +and borgmatic uses that format to name any new archive it creates. For +instance: + +```yaml +location: + ... + archive_name_format: home-directories-{now} +``` + +This means that when borgmatic creates an archive, its name will start with +the string `home-directories-` and end with a timestamp for its creation time. +If `archive_name_format` is unspecified, the default is +`{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a +timestamp in a particular format. + +New in version 1.7.11 borgmatic +uses the `archive_name_format` option to automatically limit which archives +get used for actions operating on multiple archives. This prevents, for +instance, duplicate archives from showing up in `rlist` or `info` results—even +if the same repository appears in multiple borgmatic configuration files. To +take advantage of this feature, simply use a different `archive_name_format` +in each configuration file. + +Under the hood, borgmatic accomplishes this by substituting globs for certain +ephemeral data placeholders in your `archive_name_format`—and using the result +to filter archives for supported actions. + +For instance, let's say that you have this in your configuration: + +```yaml +location: + ... + archive_name_format: {hostname}-user-data-{now} +``` + +borgmatic considers `{now}` an emphemeral placeholder that will probably +change per archive, while `{hostname}` won't. So it turns the example value +into `{hostname}-user-data-*` and applies it to filter down the set of +archives used for actions like `rlist`, `info`, `prune`, `check`, etc. + +The end result is that when borgmatic runs the actions for a particular +application-specific configuration file, it only operates on the archives +created for that application. Of course, this doesn't apply to actions like +`compact` that operate on an entire repository. + +Prior to 1.7.11 The way to +limit the archives used was a `prefix` option in the `retention` section for +matching against the start of archive names used for a `prune` action and +a separate `prefix` option in the `consistency` section for matching against +the start of archive names used for a `check` action. Both of these options +are deprecated in favor of the auto-matching behavior in newer versions of +borgmatic mentioned above. + + ## Configuration includes Once you have multiple different configuration files, you might want to share @@ -272,7 +338,7 @@ Here's an example usage: ```yaml constants: user: foo - my_prefix: bar- + archive_prefix: bar location: source_directories: @@ -281,20 +347,14 @@ location: ... storage: - archive_name_format: '{my_prefix}{now}' - -retention: - prefix: {my_prefix} - -consistency: - prefix: {my_prefix} + archive_name_format: '{archive_prefix}-{now}' ``` In this example, when borgmatic runs, all instances of `{user}` get replaced -with `foo` and all instances of `{my_prefix}` get replaced with `bar-`. (And -in this particular example, `{now}` doesn't get replaced with anything, but -gets passed directly to Borg.) After substitution, the logical result looks -something like this: +with `foo` and all instances of `{archive-prefix}` get replaced with `bar-`. +(And in this particular example, `{now}` doesn't get replaced with anything, +but gets passed directly to Borg.) After substitution, the logical result +looks something like this: ```yaml location: @@ -305,12 +365,6 @@ location: storage: archive_name_format: 'bar-{now}' - -retention: - prefix: bar- - -consistency: - prefix: bar- ``` An alternate to constants is passing in your values via [environment diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 75755565..7c233bcf 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -189,150 +189,170 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_ def test_make_check_flags_with_repository_check_returns_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',)) + flags = module.make_check_flags('1.2.3', {}, ('repository',)) assert flags == ('--repository-only',) def test_make_check_flags_with_archives_check_returns_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',)) + flags = module.make_check_flags('1.2.3', {}, ('archives',)) assert flags == ('--archives-only',) def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',)) + flags = module.make_check_flags('1.2.3', {}, ('data',)) assert flags == ('--archives-only', '--verify-data',) def test_make_check_flags_with_extract_omits_extract_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('extract',)) + flags = module.make_check_flags('1.2.3', {}, ('extract',)) assert flags == () def test_make_check_flags_with_repository_and_data_checks_does_not_return_repository_only(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'data',)) + flags = module.make_check_flags('1.2.3', {}, ('repository', 'data',)) assert flags == ('--verify-data',) -def test_make_check_flags_with_default_checks_and_default_prefix_returns_default_flags(): +def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo',) + + assert flags == ('--match-archives', 'sh:foo*') + + +def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', ('repository', 'archives'), prefix=module.DEFAULT_PREFIX + '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', ) - assert flags == ('--match-archives', f'sh:{module.DEFAULT_PREFIX}*') + assert flags == ('--match-archives', 'sh:foo*') -def test_make_check_flags_with_all_checks_and_default_prefix_returns_default_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - - flags = module.make_check_flags( - '1.2.3', ('repository', 'archives', 'extract'), prefix=module.DEFAULT_PREFIX - ) - - assert flags == ('--match-archives', f'sh:{module.DEFAULT_PREFIX}*') - - -def test_make_check_flags_with_all_checks_and_default_prefix_without_borg_features_returns_glob_archives_flags(): +def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', ('repository', 'archives', 'extract'), prefix=module.DEFAULT_PREFIX + '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', ) - assert flags == ('--glob-archives', f'{module.DEFAULT_PREFIX}*') + assert flags == ('--glob-archives', 'foo*') def test_make_check_flags_with_archives_check_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('archives',), check_last=3) assert flags == ('--archives-only', '--last', '3') def test_make_check_flags_with_data_check_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('data',), check_last=3) assert flags == ('--archives-only', '--last', '3', '--verify-data') def test_make_check_flags_with_repository_check_and_last_omits_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('repository',), check_last=3) assert flags == ('--repository-only',) def test_make_check_flags_with_default_checks_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'archives'), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) assert flags == ('--last', '3') def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix='foo-') assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('data',), prefix='foo-') assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data') -def test_make_check_flags_with_archives_check_and_empty_prefix_omits_match_archives_flag(): +def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) - flags = module.make_check_flags('1.2.3', ('archives',), prefix='') + flags = module.make_check_flags( + '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 + ) - assert flags == ('--archives-only',) + assert flags == ('--archives-only', '--match-archives', 'sh:bar-*') def test_make_check_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), prefix=None) + flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix=None) assert flags == ('--archives-only',) def test_make_check_flags_with_repository_check_and_prefix_omits_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('repository',), prefix='foo-') assert flags == ('--repository-only',) def test_make_check_flags_with_default_checks_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'archives'), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') assert flags == ('--match-archives', 'sh:foo-*') @@ -427,7 +447,7 @@ def test_check_archives_calls_borg_with_parameters(checks): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', {}, checks, check_last, prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) @@ -581,7 +601,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', {}, checks, check_last, prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1', 'check', 'repo')) @@ -608,7 +628,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', {}, checks, check_last, prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo')) @@ -628,6 +648,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): checks = ('repository',) check_last = flexmock() + storage_config = {'lock_wait': 5} consistency_config = {'check_last': check_last} flexmock(module).should_receive('parse_checks') flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) @@ -635,7 +656,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', storage_config, checks, check_last, None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo')) @@ -645,7 +666,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.check_archives( repository_path='repo', location_config={}, - storage_config={'lock_wait': 5}, + storage_config=storage_config, consistency_config=consistency_config, local_borg_version='1.2.3', ) @@ -662,7 +683,7 @@ def test_check_archives_with_retention_prefix(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, prefix + '1.2.3', {}, checks, check_last, prefix ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 1985d816..cf9eedbb 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -1,3 +1,4 @@ +import pytest from flexmock import flexmock from borgmatic.borg import flags as module @@ -78,3 +79,35 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a assert module.make_repository_archive_flags( repository_path='repo', archive='archive', local_borg_version='1.2.3' ) == ('repo::archive',) + + +@pytest.mark.parametrize( + 'archive_name_format,feature_available,expected_result', + ( + (None, True, ()), + ('', True, ()), + ( + '{hostname}-docs-{now}', # noqa: FS003 + True, + ('--match-archives', 'sh:{hostname}-docs-*'), # noqa: FS003 + ), + ('{utcnow}-docs-{user}', True, ('--match-archives', 'sh:*-docs-{user}')), # noqa: FS003 + ('{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003 + ( + 'stuff-{now:%Y-%m-%dT%H:%M:%S.%f}', # noqa: FS003 + True, + ('--match-archives', 'sh:stuff-*'), + ), + ('{hostname}-docs-{now}', False, ('--glob-archives', '{hostname}-docs-*')), # noqa: FS003 + ('{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003 + ), +) +def test_make_match_archives_flags_makes_flags_with_globs( + archive_name_format, feature_available, expected_result +): + flexmock(module.feature).should_receive('available').and_return(feature_available) + + assert ( + module.make_match_archives_flags(archive_name_format, local_borg_version=flexmock()) + == expected_result + ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index fcc556a4..b25f29f6 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -12,6 +12,9 @@ def test_display_archives_info_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -34,6 +37,9 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -56,6 +62,9 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -78,6 +87,9 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -101,6 +113,9 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -123,6 +138,9 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -147,6 +165,9 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'archive' ).and_return(('--match-archives', 'archive')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -169,6 +190,9 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -195,6 +219,9 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para flexmock(module.flags).should_receive('make_flags').with_args( 'remote-path', 'borg1' ).and_return(('--remote-path', 'borg1')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -221,6 +248,9 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete flexmock(module.flags).should_receive('make_flags').with_args('lock-wait', 5).and_return( ('--lock-wait', '5') ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) storage_config = {'lock_wait': 5} @@ -240,13 +270,16 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete ) -def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parameters(): +def test_display_archives_info_transforms_prefix_into_match_archives_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -265,12 +298,68 @@ def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parame ) +def test_display_archives_info_prefers_prefix_over_archive_name_format(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args( + 'match-archives', 'sh:foo*' + ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix='foo'), + ) + + +def test_display_archives_info_transforms_archive_name_format_into_match_archives_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:bar-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None), + ) + + @pytest.mark.parametrize('argument_name', ('match_archives', 'sort_by', 'first', 'last')) def test_display_archives_info_passes_through_arguments_to_borg(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index f33c9933..1c8843e2 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -18,18 +18,17 @@ def insert_execute_command_mock(prune_command, output_log_level): ).once() -BASE_PRUNE_FLAGS = (('--keep-daily', '1'), ('--keep-weekly', '2'), ('--keep-monthly', '3')) +BASE_PRUNE_FLAGS = ('--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') -def test_make_prune_flags_returns_flags_from_config_plus_default_prefix_glob(): +def test_make_prune_flags_returns_flags_from_config(): retention_config = OrderedDict((('keep_daily', 1), ('keep_weekly', 2), ('keep_monthly', 3))) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') - assert tuple(result) == BASE_PRUNE_FLAGS + ( - ('--match-archives', 'sh:{hostname}-*'), # noqa: FS003 - ) + assert result == BASE_PRUNE_FLAGS def test_make_prune_flags_accepts_prefix_with_placeholders(): @@ -37,15 +36,18 @@ def test_make_prune_flags_accepts_prefix_with_placeholders(): (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 ) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') expected = ( - ('--keep-daily', '1'), - ('--match-archives', 'sh:Documents_{hostname}-{now}*'), # noqa: FS003 + '--keep-daily', + '1', + '--match-archives', + 'sh:Documents_{hostname}-{now}*', # noqa: FS003 ) - assert tuple(result) == expected + assert result == expected def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives(): @@ -53,37 +55,38 @@ def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives() (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 ) flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') expected = ( - ('--keep-daily', '1'), - ('--glob-archives', 'Documents_{hostname}-{now}*'), # noqa: FS003 + '--keep-daily', + '1', + '--glob-archives', + 'Documents_{hostname}-{now}*', # noqa: FS003 ) - assert tuple(result) == expected + assert result == expected -def test_make_prune_flags_treats_empty_prefix_as_no_prefix(): - retention_config = OrderedDict((('keep_daily', 1), ('prefix', ''))) - flexmock(module.feature).should_receive('available').and_return(True) - - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') - - expected = (('--keep-daily', '1'),) - - assert tuple(result) == expected - - -def test_make_prune_flags_treats_none_prefix_as_no_prefix(): +def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): + storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 retention_config = OrderedDict((('keep_daily', 1), ('prefix', None))) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') - expected = (('--keep-daily', '1'),) + expected = ( + '--keep-daily', + '1', + '--match-archives', + 'sh:bar-*', # noqa: FS003 + ) - assert tuple(result) == expected + assert result == expected PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index a098cafa..8a10b075 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -127,6 +127,9 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter def test_make_rlist_command_includes_log_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -143,6 +146,9 @@ def test_make_rlist_command_includes_log_info(): def test_make_rlist_command_includes_json_but_not_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -159,6 +165,9 @@ def test_make_rlist_command_includes_json_but_not_info(): def test_make_rlist_command_includes_log_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -175,6 +184,9 @@ def test_make_rlist_command_includes_log_debug(): def test_make_rlist_command_includes_json_but_not_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -190,6 +202,9 @@ def test_make_rlist_command_includes_json_but_not_debug(): def test_make_rlist_command_includes_json(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -207,6 +222,9 @@ def test_make_rlist_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -222,6 +240,9 @@ def test_make_rlist_command_includes_lock_wait(): def test_make_rlist_command_includes_local_path(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -240,6 +261,9 @@ def test_make_rlist_command_includes_remote_path(): flexmock(module.flags).should_receive('make_flags').and_return( ('--remote-path', 'borg2') ).and_return(()).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -258,6 +282,9 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( ('--match-archives', 'sh:foo*') ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -271,8 +298,47 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') +def test_make_rlist_command_prefers_prefix_over_archive_name_format(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( + ('--match-archives', 'sh:foo*') + ) + flexmock(module.flags).should_receive('make_match_archives_flags').never() + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + ) + + assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') + + +def test_make_rlist_command_transforms_archive_name_format_into_match_archives(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + ) + + assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') + + def test_make_rlist_command_includes_short(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--short',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -301,6 +367,9 @@ def test_make_rlist_command_includes_short(): ) def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (f"--{argument_name.replace('_', '-')}", 'value') ) diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index a4814420..3628a1dc 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -12,6 +12,7 @@ def test_transfer_archives_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -41,6 +42,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): flexmock(module.flags).should_receive('make_flags').with_args('dry-run', True).and_return( ('--dry-run',) ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -67,6 +69,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -93,6 +96,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -123,6 +127,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'archive' ).and_return(('--match-archives', 'archive')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -137,7 +142,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive='archive', progress=None, match_archives=None, source_repository=None @@ -152,6 +157,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -166,7 +172,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None @@ -174,10 +180,40 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl ) +def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archives_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--match-archives', 'sh:bar-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, + borg_local_path='borg', + extra_environment=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + ) + + def test_transfer_archives_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -208,6 +244,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.flags).should_receive('make_flags').with_args( 'remote-path', 'borg2' ).and_return(('--remote-path', 'borg2')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -238,6 +275,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.flags).should_receive('make_flags').with_args('lock-wait', 5).and_return( ('--lock-wait', '5') ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) storage_config = {'lock_wait': 5} @@ -265,6 +303,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -293,6 +332,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') ) @@ -327,6 +367,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla flexmock(module.flags).should_receive('make_flags').with_args('other-repo', 'other').and_return( ('--other-repo', 'other') ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') From f256908b2709b643b4f7b27fd3fda150d3d32bca Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 31 Mar 2023 15:36:59 -0700 Subject: [PATCH 039/344] Document wording tweaks (#479). --- docs/how-to/make-per-application-backups.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 6e5d999a..fffd2c9d 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -91,7 +91,7 @@ in each configuration file. Under the hood, borgmatic accomplishes this by substituting globs for certain ephemeral data placeholders in your `archive_name_format`—and using the result -to filter archives for supported actions. +to filter archives when running supported actions. For instance, let's say that you have this in your configuration: @@ -101,7 +101,7 @@ location: archive_name_format: {hostname}-user-data-{now} ``` -borgmatic considers `{now}` an emphemeral placeholder that will probably +borgmatic considers `{now}` an emphemeral data placeholder that will probably change per archive, while `{hostname}` won't. So it turns the example value into `{hostname}-user-data-*` and applies it to filter down the set of archives used for actions like `rlist`, `info`, `prune`, `check`, etc. @@ -112,12 +112,11 @@ created for that application. Of course, this doesn't apply to actions like `compact` that operate on an entire repository. Prior to 1.7.11 The way to -limit the archives used was a `prefix` option in the `retention` section for -matching against the start of archive names used for a `prune` action and -a separate `prefix` option in the `consistency` section for matching against -the start of archive names used for a `check` action. Both of these options -are deprecated in favor of the auto-matching behavior in newer versions of -borgmatic mentioned above. +limit the archives used for the `prune` action was a `prefix` option in the +`retention` section for matching against the start of archive names. And the +option for limiting the archives used for the `check` action was a separate +`prefix` in the `consistency` section. Both of these options are deprecated in +favor of the auto-matching behavior in newer versions of borgmatic. ## Configuration includes From fa8bc285c8e82d8fd89327f5877c95957a8b4c76 Mon Sep 17 00:00:00 2001 From: kxxt Date: Sat, 1 Apr 2023 13:45:32 +0800 Subject: [PATCH 040/344] Fix randomly failing test. --- tests/integration/test_execute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 09e9093a..9c62941b 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -147,7 +147,7 @@ def test_log_outputs_kills_other_processes_when_one_errors(): ['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) flexmock(module).should_receive('exit_code_indicates_error').with_args( - other_process, None, 'borg' + ['sleep', '2'], None, 'borg' ).and_return(False) flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return( process.stdout From 6054ced931d35173627019054d9bc801cfd2d1ed Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 1 Apr 2023 22:10:32 +0530 Subject: [PATCH 041/344] fix: run typos --- NEWS | 6 +++--- borgmatic/borg/prune.py | 2 +- borgmatic/commands/arguments.py | 2 +- borgmatic/config/convert.py | 2 +- borgmatic/config/schema.yaml | 4 ++-- borgmatic/logger.py | 2 +- docs/_includes/components/toc.css | 2 +- ...backup-to-a-removable-drive-or-an-intermittent-server.md | 4 ++-- docs/how-to/develop-on-borgmatic.md | 4 ++-- docs/how-to/inspect-your-backups.md | 2 +- docs/how-to/set-up-backups.md | 2 +- tests/integration/config/test_validate.py | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/NEWS b/NEWS index 9bc9ebf6..c0ae3d26 100644 --- a/NEWS +++ b/NEWS @@ -388,7 +388,7 @@ configuration schema descriptions. 1.5.6 - * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the + * #292: Allow before_backup and similar hooks to exit with a soft failure without altering the monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring services with a "start" status until after before_* hooks finish. Failures in before_* hooks still trigger a monitoring "fail" status. @@ -457,7 +457,7 @@ * For "list" and "info" actions, show repository names even at verbosity 0. 1.4.22 - * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON ouput. + * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON output. * After a backup of a database dump in directory format, properly remove the dump directory. * In "borgmatic --help", don't expand $HOME in listing of default "--config" paths. @@ -829,7 +829,7 @@ * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files, editor swap files, etc. * #81: Document user-defined hooks run before/after backup, or on error. - * Add code style guidelines to the documention. + * Add code style guidelines to the documentation. 1.2.0 * #61: Support for Borg --list option via borgmatic command-line to list all archives. diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index d21ceee3..e9f665f8 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def make_prune_flags(retention_config, local_borg_version): ''' - Given a retention config dict mapping from option name to value, tranform it into an iterable of + Given a retention config dict mapping from option name to value, transform it into an iterable of command-line name-value flag pairs. For example, given a retention config of: diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 59751b08..44e11ebb 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -293,7 +293,7 @@ def make_parsers(): ) transfer_group.add_argument( '--upgrader', - help='Upgrader type used to convert the transfered data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion', + help='Upgrader type used to convert the transferred data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion', ) transfer_group.add_argument( '--progress', diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index e4e55e43..093ad0c1 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -43,7 +43,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): ] ) - # Split space-seperated values into actual lists, make "repository" into a list, and merge in + # Split space-separated values into actual lists, make "repository" into a list, and merge in # excludes. location = destination_config['location'] location['source_directories'] = source_config.location['source_directories'].split(' ') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3e94b8d2..fe673191 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -6,7 +6,7 @@ properties: constants: type: object description: | - Constants to use in the configuration file. All occurences of the + Constants to use in the configuration file. All occurrences of the constant name within culy braces will be replaced with the value. For example, if you have a constant named "hostname" with the value "myhostname", then the string "{hostname}" will be replaced with @@ -929,7 +929,7 @@ properties: type: string enum: ['sql'] description: | - Database dump output format. Currenly only "sql" + Database dump output format. Currently only "sql" is supported. Defaults to "sql" for a single database. Or, when database name is "all" and format is blank, dumps all databases to a single diff --git a/borgmatic/logger.py b/borgmatic/logger.py index e098bf96..648500b0 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -68,7 +68,7 @@ class Multi_stream_handler(logging.Handler): def emit(self, record): ''' - Dispatch the log record to the approriate stream handler for the record's log level. + Dispatch the log record to the appropriate stream handler for the record's log level. ''' self.log_level_to_handler[record.levelno].emit(record) diff --git a/docs/_includes/components/toc.css b/docs/_includes/components/toc.css index 039673f6..82cf15e5 100644 --- a/docs/_includes/components/toc.css +++ b/docs/_includes/components/toc.css @@ -94,7 +94,7 @@ display: block; } -/* Footer catgory navigation */ +/* Footer category navigation */ .elv-cat-list-active { font-weight: 600; } diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index 098b03f6..04ccbf79 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -116,8 +116,8 @@ There are some caveats you should be aware of with this feature. * You'll generally want to put a soft failure command in the `before_backup` hook, so as to gate whether the backup action occurs. While a soft failure is also supported in the `after_backup` hook, returning a soft failure there - won't prevent any actions from occuring, because they've already occurred! - Similiarly, you can return a soft failure from an `on_error` hook, but at + won't prevent any actions from occurring, because they've already occurred! + Similarly, you can return a soft failure from an `on_error` hook, but at that point it's too late to prevent the error. * Returning a soft failure does prevent further commands in the same hook from executing. So, like a standard error, it is an "early out". Unlike a standard diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index d4315457..b615ed00 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -101,10 +101,10 @@ the following deviations from it: * For strings, prefer single quotes over double quotes. * Limit all lines to a maximum of 100 characters. * Use trailing commas within multiline values or argument lists. - * For multiline constructs, put opening and closing delimeters on lines + * For multiline constructs, put opening and closing delimiters on lines separate from their contents. * Within multiline constructs, use standard four-space indentation. Don't align - indentation with an opening delimeter. + indentation with an opening delimiter. borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 57a2381c..67d8cc23 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -111,7 +111,7 @@ By default, borgmatic logs to a local syslog-compatible daemon if one is present and borgmatic is running in a non-interactive console. Where those logs show up depends on your particular system. If you're using systemd, try running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or -similiar. +similar. You can customize the log level used for syslog logging with the `--syslog-verbosity` flag, and this is independent from the console logging diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 52962c34..fb3eb91f 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -157,7 +157,7 @@ variable or set the `BORG_PASSPHRASE` environment variable. See the section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption) of the Borg Quick Start for more info. -Alternatively, you can specify the passphrase programatically by setting +Alternatively, you can specify the passphrase programmatically by setting either the borgmatic `encryption_passcommand` configuration variable or the `BORG_PASSCOMMAND` environment variable. See the [Borg Security FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically) diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 67e510ab..87428ddc 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -8,7 +8,7 @@ from flexmock import flexmock from borgmatic.config import validate as module -def test_schema_filename_returns_plausable_path(): +def test_schema_filename_returns_plausible_path(): schema_path = module.schema_filename() assert schema_path.endswith('/schema.yaml') From 2934d0902c8ec26b4d1cc2c93e4a46f7801eeb67 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Apr 2023 11:03:59 -0700 Subject: [PATCH 042/344] Code spell checking on every test run! --- borgmatic/borg/check.py | 2 +- borgmatic/config/schema.yaml | 16 ++++++++-------- docs/how-to/develop-on-borgmatic.md | 8 ++++++++ setup.cfg | 3 +++ test_requirements.txt | 1 + tox.ini | 6 ++++++ 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 4630782a..2c7c8fc5 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -147,7 +147,7 @@ def filter_checks_on_frequency( def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None): ''' - Given the local Borg version, a storge configuration dict, a parsed sequence of checks, the + Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the check last value, and a consistency check prefix, transform the checks into tuple of command-line flags. diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 875c05c3..04b3c6ec 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -930,14 +930,14 @@ properties: type: string enum: ['sql'] description: | - Database dump output format. Currently only "sql" - is supported. Defaults to "sql" for a single - database. Or, when database name is "all" and - format is blank, dumps all databases to a single - file. But if a format is specified with an "all" - database name, dumps each database to a separate - file of that format, allowing more convenient - restores of individual databases. + Database dump output format. Currently only + "sql" is supported. Defaults to "sql" for a + single database. Or, when database name is "all" + and format is blank, dumps all databases to a + single file. But if a format is specified with + an "all" database name, dumps each database to a + separate file of that format, allowing more + convenient restores of individual databases. example: directory add_drop_database: type: boolean diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 33346fae..ebc79c6f 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -73,6 +73,14 @@ can ask isort to order your imports for you: tox -e isort ``` +Similarly, if you get errors about spelling mistakes in source code, you can +ask codespell to correct them: + +```bash +tox -e codespell +``` + + ### End-to-end tests borgmatic additionally includes some end-to-end tests that integration test diff --git a/setup.cfg b/setup.cfg index 67b989dc..f97ae972 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,3 +20,6 @@ known_first_party = borgmatic line_length = 100 multi_line_output = 3 skip = .tox + +[codespell] +skip = .git,.tox,build diff --git a/test_requirements.txt b/test_requirements.txt index d34bc623..7910788b 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -2,6 +2,7 @@ appdirs==1.4.4; python_version >= '3.8' attrs==20.3.0; python_version >= '3.8' black==19.10b0; python_version >= '3.8' click==7.1.2; python_version >= '3.8' +codespell==2.2.4 colorama==0.4.4 coverage==5.3 flake8==4.0.1 diff --git a/tox.ini b/tox.ini index 17b7a9da..b81588af 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,7 @@ commands = py38,py39,py310,py311: black --check . isort --check-only --settings-path setup.cfg . flake8 borgmatic tests + codespell [testenv:black] commands = @@ -35,3 +36,8 @@ commands = deps = {[testenv]deps} commands = isort --settings-path setup.cfg . + +[testenv:codespell] +deps = {[testenv]deps} +commands = + codespell --write-changes From b9328e6d4273a3ad7a1ef4d990888584964f0a0c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Apr 2023 14:09:48 -0700 Subject: [PATCH 043/344] Add spellchecking of source code to NEWS. --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index cce27406..af0fe9ac 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,7 @@ auto-matching behavior (see above). * #662: Fix regression in which "check_repositories" option failed to match repositories. * #663: Fix regression in which the "transfer" action produced a traceback. + * Add spellchecking of source code during test runs. 1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. From 275e99d0b9dcbe0ab3509bdbbd10df1dad37c10c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Apr 2023 14:38:52 -0700 Subject: [PATCH 044/344] Add codespell link to documentation. --- docs/how-to/develop-on-borgmatic.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index ebc79c6f..6699bdf3 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -74,7 +74,8 @@ tox -e isort ``` Similarly, if you get errors about spelling mistakes in source code, you can -ask codespell to correct them: +ask [codespell](https://github.com/codespell-project/codespell) to correct +them: ```bash tox -e codespell From 9712d00680fd0f8ec77e506d561745161e9cac04 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Apr 2023 23:57:55 -0700 Subject: [PATCH 045/344] Add "match_archives" option (#588). --- NEWS | 11 ++++--- borgmatic/borg/check.py | 4 ++- borgmatic/borg/flags.py | 21 ++++++++----- borgmatic/borg/info.py | 4 ++- borgmatic/borg/prune.py | 4 ++- borgmatic/borg/rlist.py | 4 ++- borgmatic/borg/transfer.py | 4 ++- borgmatic/config/schema.yaml | 11 +++++++ docs/how-to/make-per-application-backups.md | 20 +++++++++++- tests/unit/borg/test_check.py | 2 +- tests/unit/borg/test_flags.py | 35 +++++++++++++++------ tests/unit/borg/test_info.py | 28 ++++++++--------- tests/unit/borg/test_prune.py | 2 +- tests/unit/borg/test_rlist.py | 24 +++++++------- tests/unit/borg/test_transfer.py | 2 +- 15 files changed, 120 insertions(+), 56 deletions(-) diff --git a/NEWS b/NEWS index af0fe9ac..739ce86c 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,11 @@ 1.7.11.dev0 - * #479: Automatically use the "archive_name_format" option to filter which archives get used for - borgmatic actions that operate on multiple archives. See the documentation for more information: + * #479, #588: Automatically use the "archive_name_format" option to filter which archives get used + for borgmatic actions that operate on multiple archives. Override this behavior with the new + "match_archives" option in the storage section. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming - * #479: The "prefix" options have been deprecated in favor of the new "archive_name_format" - auto-matching behavior (see above). - * #662: Fix regression in which "check_repositories" option failed to match repositories. + * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format" + auto-matching behavior and the "match_archives" option. + * #662: Fix regression in which the "check_repositories" option failed to match repositories. * #663: Fix regression in which the "transfer" action produced a traceback. * Add spellchecking of source code during test runs. diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 2c7c8fc5..cee9d923 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -183,7 +183,9 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None if prefix else ( flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) ) ) diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 1354d658..986531bc 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -59,18 +59,25 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version): ) -def make_match_archives_flags(archive_name_format, local_borg_version): +def make_match_archives_flags(match_archives, archive_name_format, local_borg_version): ''' - Return the match archives flags that would match archives created with the given archive name - format (if any). This is done by replacing certain archive name format placeholders for - ephemeral data (like "{now}") with globs. + Return match archives flags based on the given match archives value, if any. If it isn't set, + return match archives flags to match archives created with the given archive name format, if + any. This is done by replacing certain archive name format placeholders for ephemeral data (like + "{now}") with globs. ''' + if match_archives: + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): + return ('--match-archives', match_archives) + else: + return ('--glob-archives', re.sub(r'^sh:', '', match_archives)) + if not archive_name_format: return () - match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format) + derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format) if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): - return ('--match-archives', f'sh:{match_archives}') + return ('--match-archives', f'sh:{derived_match_archives}') else: - return ('--glob-archives', f'{match_archives}') + return ('--glob-archives', f'{derived_match_archives}') diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index fa59165c..a7ed98ed 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -46,7 +46,9 @@ def display_archives_info( if info_arguments.prefix else ( flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) ) ) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 11ba308e..7e12b9c2 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -39,7 +39,9 @@ def make_prune_flags(storage_config, retention_config, local_borg_version): return tuple( element for pair in flag_pairs for element in pair ) + flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index f3935a16..f84499ce 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -96,7 +96,9 @@ def make_rlist_command( if rlist_arguments.prefix else ( flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) ) ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index a350eff7..78a76124 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -41,7 +41,9 @@ def transfer_archives( ) or ( flags.make_match_archives_flags( - storage_config.get('archive_name_format'), local_borg_version + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, ) ) ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 04b3c6ec..26500855 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -382,6 +382,17 @@ properties: actions like rlist, info, or check, borgmatic automatically tries to match only archives created with this name format. example: "{hostname}-documents-{now}" + match_archives: + type: string + description: | + A Borg pattern for filtering down the archives used by + borgmatic actions that operate on multiple archives. For + Borg 1.x, use a shell pattern here and see the output of + "borg help placeholders" for details. For Borg 2.x, see the + output of "borg help match-archives". If match_archives is + not specified, borgmatic defaults to deriving the + match_archives value from archive_name_format. + example: "sh:{hostname}-*" relocated_repo_access_is_ok: type: boolean description: | diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index fffd2c9d..a4898430 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -111,12 +111,30 @@ application-specific configuration file, it only operates on the archives created for that application. Of course, this doesn't apply to actions like `compact` that operate on an entire repository. +If this behavior isn't quite smart enough for your needs, you can use the +`match_archives` option to override the pattern that borgmatic uses for +filtering archives. For example: + +```yaml +location: + ... + archive_name_format: {hostname}-user-data-{now} + match_archives: sh:myhost-user-data-* +``` + +For Borg 1.x, use a shell pattern for the `match_archives` value and see the +[Borg patterns +documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns) +for more information. For Borg 2.x, see the [match archives +documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/help.html#borg-help-match-archives). + Prior to 1.7.11 The way to limit the archives used for the `prune` action was a `prefix` option in the `retention` section for matching against the start of archive names. And the option for limiting the archives used for the `check` action was a separate `prefix` in the `consistency` section. Both of these options are deprecated in -favor of the auto-matching behavior in newer versions of borgmatic. +favor of the auto-matching behavior (or `match_archives`) in newer versions of +borgmatic. ## Configuration includes diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 7c233bcf..65e17291 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -320,7 +320,7 @@ def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_fla def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '1.2.3' # noqa: FS003 + None, 'bar-{now}', '1.2.3' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) flags = module.make_check_flags( diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index cf9eedbb..86247040 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -82,32 +82,49 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a @pytest.mark.parametrize( - 'archive_name_format,feature_available,expected_result', + 'match_archives, archive_name_format,feature_available,expected_result', ( - (None, True, ()), - ('', True, ()), + (None, None, True, ()), + (None, '', True, ()), + ('re:foo-.*', '{hostname}-{now}', True, ('--match-archives', 're:foo-.*'),), # noqa: FS003 + ('sh:foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003 + ('foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003 ( + None, '{hostname}-docs-{now}', # noqa: FS003 True, ('--match-archives', 'sh:{hostname}-docs-*'), # noqa: FS003 ), - ('{utcnow}-docs-{user}', True, ('--match-archives', 'sh:*-docs-{user}')), # noqa: FS003 - ('{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003 ( + None, + '{utcnow}-docs-{user}', # noqa: FS003 + True, + ('--match-archives', 'sh:*-docs-{user}'), # noqa: FS003 + ), + (None, '{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003 + ( + None, 'stuff-{now:%Y-%m-%dT%H:%M:%S.%f}', # noqa: FS003 True, ('--match-archives', 'sh:stuff-*'), ), - ('{hostname}-docs-{now}', False, ('--glob-archives', '{hostname}-docs-*')), # noqa: FS003 - ('{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003 + ( + None, + '{hostname}-docs-{now}', # noqa: FS003 + False, + ('--glob-archives', '{hostname}-docs-*'), # noqa: FS003 + ), + (None, '{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003 ), ) def test_make_match_archives_flags_makes_flags_with_globs( - archive_name_format, feature_available, expected_result + match_archives, archive_name_format, feature_available, expected_result ): flexmock(module.feature).should_receive('available').and_return(feature_available) assert ( - module.make_match_archives_flags(archive_name_format, local_borg_version=flexmock()) + module.make_match_archives_flags( + match_archives, archive_name_format, local_borg_version=flexmock() + ) == expected_result ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index b25f29f6..ee525857 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -13,7 +13,7 @@ def test_display_archives_info_calls_borg_with_parameters(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -38,7 +38,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -63,7 +63,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -88,7 +88,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -114,7 +114,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -139,7 +139,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -166,7 +166,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param 'match-archives', 'archive' ).and_return(('--match-archives', 'archive')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -191,7 +191,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -220,7 +220,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para 'remote-path', 'borg1' ).and_return(('--remote-path', 'borg1')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -249,7 +249,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete ('--lock-wait', '5') ) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -278,7 +278,7 @@ def test_display_archives_info_transforms_prefix_into_match_archives_parameters( 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -306,7 +306,7 @@ def test_display_archives_info_prefers_prefix_over_archive_name_format(): 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -331,7 +331,7 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '2.3.4' # noqa: FS003 + None, 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) @@ -358,7 +358,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '2.3.4' + None, None, '2.3.4' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 1c8843e2..37731aca 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -74,7 +74,7 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): retention_config = OrderedDict((('keep_daily', 1), ('prefix', None))) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '1.2.3' # noqa: FS003 + None, 'bar-{now}', '1.2.3' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 8a10b075..136821d6 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -128,7 +128,7 @@ def test_make_rlist_command_includes_log_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -147,7 +147,7 @@ def test_make_rlist_command_includes_json_but_not_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -166,7 +166,7 @@ def test_make_rlist_command_includes_log_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -185,7 +185,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -203,7 +203,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): def test_make_rlist_command_includes_json(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -223,7 +223,7 @@ def test_make_rlist_command_includes_lock_wait(): ('--lock-wait', '5') ).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -241,7 +241,7 @@ def test_make_rlist_command_includes_lock_wait(): def test_make_rlist_command_includes_local_path(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -262,7 +262,7 @@ def test_make_rlist_command_includes_remote_path(): ('--remote-path', 'borg2') ).and_return(()).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -283,7 +283,7 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): ('--match-archives', 'sh:foo*') ) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -319,7 +319,7 @@ def test_make_rlist_command_prefers_prefix_over_archive_name_format(): def test_make_rlist_command_transforms_archive_name_format_into_match_archives(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '1.2.3' # noqa: FS003 + None, 'bar-{now}', '1.2.3' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -337,7 +337,7 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives() def test_make_rlist_command_includes_short(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--short',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -368,7 +368,7 @@ def test_make_rlist_command_includes_short(): def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, '1.2.3' + None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (f"--{argument_name.replace('_', '-')}", 'value') diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 3628a1dc..857c86ec 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -185,7 +185,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'bar-{now}', '2.3.4' # noqa: FS003 + None, 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) From 01811e03ba1c46cbf04acbbe3bde99b3433e57f9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 2 Apr 2023 14:38:35 -0700 Subject: [PATCH 046/344] Tagged the auto-matching archive behavior as breaking in NEWS. --- NEWS | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 739ce86c..908256a3 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,9 @@ 1.7.11.dev0 - * #479, #588: Automatically use the "archive_name_format" option to filter which archives get used - for borgmatic actions that operate on multiple archives. Override this behavior with the new - "match_archives" option in the storage section. See the documentation for more information: + * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives + get used for borgmatic actions that operate on multiple archives. Override this behavior with the + new "match_archives" option in the storage section. This change is "breaking" in that it silently + changes which archives get considered for "rlist", "prune", "check", etc. See the documentation + for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format" auto-matching behavior and the "match_archives" option. From 7e6bee84b0aa99aa0dece40fd6ee416d1c038674 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 2 Apr 2023 23:06:36 -0700 Subject: [PATCH 047/344] Add "--log-file-format" flag for customizing the log message format (#658). --- NEWS | 3 +++ borgmatic/commands/arguments.py | 8 +++--- borgmatic/commands/borgmatic.py | 1 + borgmatic/logger.py | 11 +++++++-- docs/how-to/inspect-your-backups.md | 38 +++++++++++++++++++++++++++-- tests/unit/test_logger.py | 33 +++++++++++++++++++++++-- 6 files changed, 85 insertions(+), 9 deletions(-) diff --git a/NEWS b/NEWS index 908256a3..dd0a6b65 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,9 @@ https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format" auto-matching behavior and the "match_archives" option. + * #658: Add "--log-file-format" flag for customizing the log message format. See the documentation + for more information: + https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#logging-to-file * #662: Fix regression in which the "check_repositories" option failed to match repositories. * #663: Fix regression in which the "transfer" action produced a traceback. * Add spellchecking of source code during test runs. diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 44e11ebb..84ab79fc 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -178,10 +178,12 @@ def make_parsers(): help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)', ) global_group.add_argument( - '--log-file', + '--log-file', type=str, help='Write log messages to this file instead of syslog', + ) + global_group.add_argument( + '--log-file-format', type=str, - default=None, - help='Write log messages to this file instead of syslog', + help='Log format string used for log messages written to the log file', ) global_group.add_argument( '--override', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b4aadfca..22aba0b1 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -700,6 +700,7 @@ def main(): # pragma: no cover verbosity_to_log_level(global_arguments.log_file_verbosity), verbosity_to_log_level(global_arguments.monitoring_verbosity), global_arguments.log_file, + global_arguments.log_file_format, ) except (FileNotFoundError, PermissionError) as error: configure_logging(logging.CRITICAL) diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 648500b0..52065928 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -156,6 +156,7 @@ def configure_logging( log_file_log_level=None, monitoring_log_level=None, log_file=None, + log_file_format=None, ): ''' Configure logging to go to both the console and (syslog or log file). Use the given log levels, @@ -200,12 +201,18 @@ def configure_logging( if syslog_path and not interactive_console(): syslog_handler = logging.handlers.SysLogHandler(address=syslog_path) - syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s')) + syslog_handler.setFormatter( + logging.Formatter('borgmatic: {levelname} {message}', style='{') # noqa: FS003 + ) syslog_handler.setLevel(syslog_log_level) handlers = (console_handler, syslog_handler) elif log_file: file_handler = logging.handlers.WatchedFileHandler(log_file) - file_handler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s')) + file_handler.setFormatter( + logging.Formatter( + log_file_format or '[{asctime}] {levelname}: {message}', style='{' # noqa: FS003 + ) + ) file_handler.setLevel(log_file_log_level) handlers = (console_handler, file_handler) else: diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 67d8cc23..b6238fe9 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -154,5 +154,39 @@ borgmatic --log-file /path/to/file.log Note that if you use the `--log-file` flag, you are responsible for rotating the log file so it doesn't grow too large, for example with -[logrotate](https://wiki.archlinux.org/index.php/Logrotate). Also, there is a -`--log-file-verbosity` flag to customize the log file's log level. +[logrotate](https://wiki.archlinux.org/index.php/Logrotate). + +You can the `--log-file-verbosity` flag to customize the log file's log level: + +```bash +borgmatic --log-file /path/to/file.log --log-file-verbosity 2 +``` + +New in borgmatic version 1.7.11 +Use the `--log-file-format` flag to override the default log message format. +This format string can contain a series of named placeholders wrapped in curly +brackets. For instance, the default log format is: `[{asctime}] {levelname}: +{message}`. This means each log message is recorded as the log time (in square +brackets), a logging level name, a colon, and the actual log message. + +So if you just want each log message to get logged *without* a timestamp or a +logging level name: + +```bash +borgmatic --log-file /path/to/file.log --log-file-format "{message}" +``` + +Here is a list of available placeholders: + + * `{asctime}`: time the log message was created + * `{levelname}`: level of the log message (`INFO`, `DEBUG`, etc.) + * `{lineno}`: line number in the source file where the log message originated + * `{message}`: actual log message + * `{pathname}`: path of the source file where the log message originated + +See the [Python logging +documentation](https://docs.python.org/3/library/logging.html#logrecord-attributes) +for additional placeholders. + +Note that this `--log-file-format` flg only applies to the specified +`--log-file` and not to syslog or other logging. diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 0fb284a3..dc9c748d 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -285,7 +285,7 @@ def test_configure_logging_skips_syslog_if_interactive_console(): module.configure_logging(console_log_level=logging.INFO) -def test_configure_logging_to_logfile_instead_of_syslog(): +def test_configure_logging_to_log_file_instead_of_syslog(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER flexmock(module).should_receive('Multi_stream_handler').and_return( @@ -309,7 +309,36 @@ def test_configure_logging_to_logfile_instead_of_syslog(): ) -def test_configure_logging_skips_logfile_if_argument_is_none(): +def test_configure_logging_to_log_file_formats_with_custom_log_format(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.ANSWER + flexmock(module.logging).should_receive('Formatter').with_args( + '{message}', style='{' # noqa: FS003 + ).once() + flexmock(module).should_receive('Multi_stream_handler').and_return( + flexmock(setFormatter=lambda formatter: None, setLevel=lambda level: None) + ) + + flexmock(module).should_receive('interactive_console').and_return(False) + flexmock(module.logging).should_receive('basicConfig').with_args( + level=logging.DEBUG, handlers=tuple + ) + flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True) + flexmock(module.logging.handlers).should_receive('SysLogHandler').never() + file_handler = logging.handlers.WatchedFileHandler('/tmp/logfile') + flexmock(module.logging.handlers).should_receive('WatchedFileHandler').with_args( + '/tmp/logfile' + ).and_return(file_handler).once() + + module.configure_logging( + console_log_level=logging.INFO, + log_file_log_level=logging.DEBUG, + log_file='/tmp/logfile', + log_file_format='{message}', # noqa: FS003 + ) + + +def test_configure_logging_skips_log_file_if_argument_is_none(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER flexmock(module).should_receive('Multi_stream_handler').and_return( From fc58ba5763d37c9b8f1c2d9e9fe6a275911eb2e5 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 3 Apr 2023 17:36:24 +0530 Subject: [PATCH 048/344] add favicon to documentation --- docs/_includes/layouts/base.njk | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_includes/layouts/base.njk b/docs/_includes/layouts/base.njk index be588e9b..dbb19d99 100644 --- a/docs/_includes/layouts/base.njk +++ b/docs/_includes/layouts/base.njk @@ -3,6 +3,7 @@ + {{ subtitle + ' - ' if subtitle}}{{ title }} {%- set css %} {% include 'index.css' %} From 00d1dea94ea826d8d91d5f6d936c70638889bb32 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 3 Apr 2023 16:11:25 -0700 Subject: [PATCH 049/344] Bump version for release. --- NEWS | 2 +- docs/how-to/inspect-your-backups.md | 6 +++--- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index dd0a6b65..f693e25f 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.11.dev0 +1.7.11 * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives get used for borgmatic actions that operate on multiple archives. Override this behavior with the new "match_archives" option in the storage section. This change is "breaking" in that it silently diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index b6238fe9..73020eda 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -162,9 +162,9 @@ You can the `--log-file-verbosity` flag to customize the log file's log level: borgmatic --log-file /path/to/file.log --log-file-verbosity 2 ``` -New in borgmatic version 1.7.11 -Use the `--log-file-format` flag to override the default log message format. -This format string can contain a series of named placeholders wrapped in curly +New in version 1.7.11 Use the +`--log-file-format` flag to override the default log message format. This +format string can contain a series of named placeholders wrapped in curly brackets. For instance, the default log format is: `[{asctime}] {levelname}: {message}`. This means each log message is recorded as the log time (in square brackets), a logging level name, a colon, and the actual log message. diff --git a/setup.py b/setup.py index bc4e4b75..ad58f0e5 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.11.dev0' +VERSION = '1.7.11' setup( From 616eb6b6daa5827395da1533ad4423fb2ce450aa Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 4 Apr 2023 21:25:10 -0700 Subject: [PATCH 050/344] Fix error with "info --match-archives" and fix "--match-archives" overriding logic (#666). --- NEWS | 5 + borgmatic/borg/info.py | 11 +- borgmatic/borg/rlist.py | 4 +- borgmatic/borg/transfer.py | 10 +- borgmatic/commands/arguments.py | 20 +++- docs/how-to/make-per-application-backups.md | 8 +- setup.py | 2 +- tests/integration/borg/test_commands.py | 104 +++++++++++++++++++ tests/integration/commands/test_arguments.py | 14 +++ tests/unit/borg/test_info.py | 90 ++++++++++++---- tests/unit/borg/test_rlist.py | 81 +++++++++++---- tests/unit/borg/test_transfer.py | 12 +-- 12 files changed, 292 insertions(+), 69 deletions(-) create mode 100644 tests/integration/borg/test_commands.py diff --git a/NEWS b/NEWS index f693e25f..96634fd0 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.7.12.dev0 + * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the + "--match-archives" flag to correctly override the "match_archives" configuration option for + the "transfer", "list", "rlist", and "info" actions. + 1.7.11 * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives get used for borgmatic actions that operate on multiple archives. Override this behavior with the diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index a7ed98ed..80739c07 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -46,21 +46,18 @@ def display_archives_info( if info_arguments.prefix else ( flags.make_match_archives_flags( - storage_config.get('match_archives'), + info_arguments.match_archives + or info_arguments.archive + or storage_config.get('match_archives'), storage_config.get('archive_name_format'), local_borg_version, ) ) ) + flags.make_flags_from_arguments( - info_arguments, excludes=('repository', 'archive', 'prefix') + info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives') ) + flags.make_repository_flags(repository_path, local_borg_version) - + ( - flags.make_flags('match-archives', info_arguments.archive) - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) - else flags.make_flags('glob-archives', info_arguments.archive) - ) ) if info_arguments.json: diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index f84499ce..9686b67b 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -52,7 +52,7 @@ def resolve_archive_name( return latest_archive -MAKE_FLAGS_EXCLUDES = ('repository', 'prefix') +MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives') def make_rlist_command( @@ -96,7 +96,7 @@ def make_rlist_command( if rlist_arguments.prefix else ( flags.make_match_archives_flags( - storage_config.get('match_archives'), + rlist_arguments.match_archives or storage_config.get('match_archives'), storage_config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 78a76124..9fd05b76 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -28,12 +28,6 @@ def transfer_archives( + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + flags.make_flags('lock-wait', storage_config.get('lock_wait', None)) - + (('--progress',) if transfer_arguments.progress else ()) - + ( - flags.make_flags( - 'match-archives', transfer_arguments.match_archives or transfer_arguments.archive - ) - ) + ( flags.make_flags_from_arguments( transfer_arguments, @@ -41,7 +35,9 @@ def transfer_archives( ) or ( flags.make_match_archives_flags( - storage_config.get('match_archives'), + transfer_arguments.match_archives + or transfer_arguments.archive + or storage_config.get('match_archives'), storage_config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 84ab79fc..13ee3153 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -652,7 +652,7 @@ def make_parsers(): '--json', default=False, action='store_true', help='Output results as JSON' ) rlist_group.add_argument( - '-P', '--prefix', help='Only list archive names starting with this prefix' + '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix' ) rlist_group.add_argument( '-a', @@ -707,7 +707,7 @@ def make_parsers(): '--json', default=False, action='store_true', help='Output results as JSON' ) list_group.add_argument( - '-P', '--prefix', help='Only list archive names starting with this prefix' + '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix' ) list_group.add_argument( '-a', @@ -779,7 +779,9 @@ def make_parsers(): '--json', dest='json', default=False, action='store_true', help='Output results as JSON' ) info_group.add_argument( - '-P', '--prefix', help='Only show info for archive names starting with this prefix' + '-P', + '--prefix', + help='Deprecated. Only show info for archive names starting with this prefix', ) info_group.add_argument( '-a', @@ -877,7 +879,17 @@ def parse_arguments(*unparsed_arguments): and arguments['transfer'].match_archives ): raise ValueError( - 'With the transfer action, only one of --archive and --glob-archives flags can be used.' + 'With the transfer action, only one of --archive and --match-archives flags can be used.' + ) + + if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives): + raise ValueError( + 'With the list action, only one of --prefix or --match-archives flags can be used.' + ) + + if 'rlist' in arguments and (arguments['rlist'].prefix and arguments['rlist'].match_archives): + raise ValueError( + 'With the rlist action, only one of --prefix or --match-archives flags can be used.' ) if 'info' in arguments and ( diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index a4898430..27c643f9 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -128,13 +128,17 @@ documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg- for more information. For Borg 2.x, see the [match archives documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/help.html#borg-help-match-archives). +Some borgmatic command-line actions also have a `--match-archives` flag that +overrides both the auto-matching behavior and the `match_archives` +configuration option. + Prior to 1.7.11 The way to limit the archives used for the `prune` action was a `prefix` option in the `retention` section for matching against the start of archive names. And the option for limiting the archives used for the `check` action was a separate `prefix` in the `consistency` section. Both of these options are deprecated in -favor of the auto-matching behavior (or `match_archives`) in newer versions of -borgmatic. +favor of the auto-matching behavior (or `match_archives`/`--match-archives`) +in newer versions of borgmatic. ## Configuration includes diff --git a/setup.py b/setup.py index ad58f0e5..4b870eb4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.11' +VERSION = '1.7.12.dev0' setup( diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py new file mode 100644 index 00000000..4f9a6b25 --- /dev/null +++ b/tests/integration/borg/test_commands.py @@ -0,0 +1,104 @@ +import copy + +import flexmock + +import borgmatic.borg.info +import borgmatic.borg.list +import borgmatic.borg.rlist +import borgmatic.borg.transfer +import borgmatic.commands.arguments + + +def assert_command_does_not_duplicate_flags(command, *args, **kwargs): + ''' + Assert that the given Borg command sequence does not contain any duplicated flags, e.g. + "--match-archives" twice anywhere in the command. + ''' + flag_counts = {} + + for flag_name in command: + if not flag_name.startswith('--'): + continue + + if flag_name in flag_counts: + flag_counts[flag_name] += 1 + else: + flag_counts[flag_name] = 1 + + assert flag_counts == { + flag_name: 1 for flag_name in flag_counts + }, f"Duplicate flags found in: {' '.join(command)}" + + +def fuzz_argument(arguments, argument_name): + ''' + Given an argparse.Namespace instance of arguments and an argument name in it, copy the arguments + namespace and set the argument name in the copy with a fake value. Return the copied arguments. + + This is useful for "fuzzing" a unit under test by passing it each possible argument in turn, + making sure it doesn't blow up or duplicate Borg arguments. + ''' + arguments_copy = copy.copy(arguments) + value = getattr(arguments_copy, argument_name) + setattr(arguments_copy, argument_name, not value if isinstance(value, bool) else 'value') + + return arguments_copy + + +def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments( + 'transfer', '--source-repository', 'foo' + )['transfer'] + flexmock(borgmatic.borg.transfer).should_receive('execute_command').replace_with( + assert_command_does_not_duplicate_flags + ) + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.transfer.transfer_archives( + False, 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) + + +def test_make_list_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('list')['list'] + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.list.make_list_command( + 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) + + +def test_make_rlist_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('rlist')['rlist'] + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.rlist.make_rlist_command( + 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) + + +def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('info')['info'] + flexmock(borgmatic.borg.info).should_receive('execute_command_and_capture_output').replace_with( + assert_command_does_not_duplicate_flags + ) + flexmock(borgmatic.borg.info).should_receive('execute_command').replace_with( + assert_command_does_not_duplicate_flags + ) + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.info.display_archives_info( + 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 754564b2..1fc8f8c8 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -465,6 +465,20 @@ def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives ) +def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + + +def test_parse_arguments_disallows_rlist_with_both_prefix_and_match_archives(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('rlist', '--prefix', 'foo', '--match-archives', 'sh:*bar') + + def test_parse_arguments_disallows_info_with_both_archive_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index ee525857..d51bf14d 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -29,7 +29,7 @@ def test_display_archives_info_calls_borg_with_parameters(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -54,7 +54,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -77,7 +77,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=True, prefix=None), + info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) assert json_output == '[]' @@ -105,7 +105,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -128,7 +128,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=True, prefix=None), + info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) assert json_output == '[]' @@ -152,7 +152,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=True, prefix=None), + info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) assert json_output == '[]' @@ -162,17 +162,14 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags').with_args( - 'match-archives', 'archive' - ).and_return(('--match-archives', 'archive')) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, None, '2.3.4' - ).and_return(()) + 'archive', None, '2.3.4' + ).and_return(('--match-archives', 'archive')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'info', '--repo', 'repo', '--match-archives', 'archive'), + ('borg', 'info', '--match-archives', 'archive', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, borg_local_path='borg', extra_environment=None, @@ -182,7 +179,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive='archive', json=False, prefix=None), + info_arguments=flexmock(archive='archive', json=False, prefix=None, match_archives=None), ) @@ -207,7 +204,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), local_path='borg1', ) @@ -236,7 +233,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), remote_path='borg1', ) @@ -266,7 +263,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -347,11 +344,64 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) -@pytest.mark.parametrize('argument_name', ('match_archives', 'sort_by', 'first', 'last')) +def test_display_archives_with_match_archives_option_calls_borg_with_match_archives_parameter(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'sh:foo-*', 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:foo-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={ + 'archive_name_format': 'bar-{now}', # noqa: FS003 + 'match_archives': 'sh:foo-*', + }, + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), + ) + + +def test_display_archives_with_match_archives_flag_calls_borg_with_match_archives_parameter(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'sh:foo-*', 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:foo-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), + ) + + +@pytest.mark.parametrize('argument_name', ('sort_by', 'first', 'last')) def test_display_archives_info_passes_through_arguments_to_borg(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -376,5 +426,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): repository_path='repo', storage_config={}, local_borg_version='2.3.4', - info_arguments=flexmock(archive=None, json=False, prefix=None, **{argument_name: 'value'}), + info_arguments=flexmock( + archive=None, json=False, prefix=None, match_archives=None, **{argument_name: 'value'} + ), ) diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 136821d6..6b1561a2 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -137,7 +137,9 @@ def test_make_rlist_command_includes_log_info(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--info', 'repo') @@ -156,7 +158,9 @@ def test_make_rlist_command_includes_json_but_not_info(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=True, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--json', 'repo') @@ -175,7 +179,9 @@ def test_make_rlist_command_includes_log_debug(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') @@ -194,7 +200,9 @@ def test_make_rlist_command_includes_json_but_not_debug(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=True, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--json', 'repo') @@ -212,7 +220,9 @@ def test_make_rlist_command_includes_json(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=True, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--json', 'repo') @@ -232,7 +242,9 @@ def test_make_rlist_command_includes_lock_wait(): repository_path='repo', storage_config={'lock_wait': 5}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') @@ -250,7 +262,9 @@ def test_make_rlist_command_includes_local_path(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), local_path='borg2', ) @@ -271,7 +285,9 @@ def test_make_rlist_command_includes_remote_path(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), remote_path='borg2', ) @@ -328,7 +344,9 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives() repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), ) assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') @@ -346,7 +364,9 @@ def test_make_rlist_command_includes_short(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None, short=True), + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None, short=True + ), ) assert command == ('borg', 'list', '--short', 'repo') @@ -354,16 +374,7 @@ def test_make_rlist_command_includes_short(): @pytest.mark.parametrize( 'argument_name', - ( - 'match_archives', - 'sort_by', - 'first', - 'last', - 'exclude', - 'exclude_from', - 'pattern', - 'patterns_from', - ), + ('sort_by', 'first', 'last', 'exclude', 'exclude_from', 'pattern', 'patterns_from',), ) def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) @@ -384,6 +395,7 @@ def test_make_rlist_command_includes_additional_flags(argument_name): paths=None, json=False, prefix=None, + match_archives=None, find_paths=None, format=None, **{argument_name: 'value'}, @@ -393,6 +405,35 @@ def test_make_rlist_command_includes_additional_flags(argument_name): assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') +def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_parameters(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'foo-*', None, '1.2.3', + ).and_return(('--match-archives', 'foo-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock( + archive=None, + paths=None, + json=False, + prefix=None, + match_archives='foo-*', + find_paths=None, + format=None, + ), + ) + + assert command == ('borg', 'list', '--match-archives', 'foo-*', 'repo') + + def test_list_repository_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 857c86ec..8f41bf5a 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -124,10 +124,9 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags').with_args( - 'match-archives', 'archive' + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'archive', 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'archive')) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -154,10 +153,9 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags').with_args( - 'match-archives', 'sh:foo*' + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'sh:foo*', 'bar-{now}', '2.3.4' # noqa: FS003 ).and_return(('--match-archives', 'sh:foo*')) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -304,7 +302,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( From a9a65ebe54d4d915cf16ba44358de5aeb7e83818 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 4 Apr 2023 22:11:36 -0700 Subject: [PATCH 051/344] Fix integration tests to actually assert (#666). --- tests/integration/borg/test_commands.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 4f9a6b25..49cd0e96 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -69,10 +69,12 @@ def test_make_list_command_does_not_duplicate_flags_or_raise(): if argument_name.startswith('_'): continue - borgmatic.borg.list.make_list_command( + command = borgmatic.borg.list.make_list_command( 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) ) + assert_command_does_not_duplicate_flags(command) + def test_make_rlist_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments('rlist')['rlist'] @@ -81,10 +83,12 @@ def test_make_rlist_command_does_not_duplicate_flags_or_raise(): if argument_name.startswith('_'): continue - borgmatic.borg.rlist.make_rlist_command( + command = borgmatic.borg.rlist.make_rlist_command( 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) ) + assert_command_does_not_duplicate_flags(command) + def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments('info')['info'] From 9bc2322f9a38c6ab704be284563b1928c7673172 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 6 Apr 2023 02:10:36 +0530 Subject: [PATCH 052/344] feat: restore specific schemas --- borgmatic/actions/restore.py | 5 +++++ borgmatic/commands/arguments.py | 7 +++++++ borgmatic/hooks/postgresql.py | 2 ++ 3 files changed, 14 insertions(+) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index bbbb4c74..fa7b04b7 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -68,12 +68,15 @@ def restore_single_database( archive_name, hook_name, database, + schemas ): # pragma: no cover ''' Given (among other things) an archive name, a database hook name, and a configured database configuration dict, restore that database from the archive. ''' logger.info(f'{repository}: Restoring database {database["name"]}') + if schemas: + database['schemas'] = schemas dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', @@ -314,6 +317,7 @@ def run_restore( archive_name, found_hook_name or hook_name, found_database, + schemas = restore_arguments.schemas, ) # For any database that weren't found via exact matches in the hooks configuration, try to @@ -343,6 +347,7 @@ def run_restore( archive_name, found_hook_name or hook_name, database, + schemas = restore_arguments.schemas, ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 13ee3153..7f93910e 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -629,6 +629,13 @@ def make_parsers(): dest='databases', help="Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic's configuration", ) + restore_group.add_argument( + '--schema', + metavar='NAME', + nargs='+', + dest='schemas', + help="Names of schemas to restore from the database, defaults to all schemas." + ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' ) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index d4799f5f..5d951f6d 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -213,6 +213,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ('--command', 'ANALYZE') ) pg_restore_command = database.get('pg_restore_command') or 'pg_restore' + backup_schemas = ', '.join(database['schemas']) if 'schemas' in database else None restore_command = ( (psql_command if all_databases else pg_restore_command, '--no-password') + ( @@ -223,6 +224,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) + + (('--schema', backup_schemas) if backup_schemas else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) ) From 080c3afa0d43f3d8b2ffdaf23a869db71ccce03e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 5 Apr 2023 14:00:21 -0700 Subject: [PATCH 053/344] Fix documentation referring to "archive_name_format" in wrong configuration section. --- docs/how-to/make-per-application-backups.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 27c643f9..99f665bc 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -70,7 +70,7 @@ and borgmatic uses that format to name any new archive it creates. For instance: ```yaml -location: +storage: ... archive_name_format: home-directories-{now} ``` @@ -96,7 +96,7 @@ to filter archives when running supported actions. For instance, let's say that you have this in your configuration: ```yaml -location: +storage: ... archive_name_format: {hostname}-user-data-{now} ``` @@ -116,7 +116,7 @@ If this behavior isn't quite smart enough for your needs, you can use the filtering archives. For example: ```yaml -location: +storage: ... archive_name_format: {hostname}-user-data-{now} match_archives: sh:myhost-user-data-* From 192bfe46a95a39e6eb34d64dbff3d2dbcb9ecce1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 5 Apr 2023 14:58:05 -0700 Subject: [PATCH 054/344] Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set (#668). --- NEWS | 2 ++ borgmatic/borg/prune.py | 26 ++++++++++++++------------ tests/unit/borg/test_prune.py | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/NEWS b/NEWS index 96634fd0..f80169a8 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,8 @@ * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the "--match-archives" flag to correctly override the "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. + * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" + options set. 1.7.11 * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 7e12b9c2..3f06dc2d 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -26,22 +26,24 @@ def make_prune_flags(storage_config, retention_config, local_borg_version): config = retention_config.copy() prefix = config.pop('prefix', None) - if prefix: - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): - config['match_archives'] = f'sh:{prefix}*' - else: - config['glob_archives'] = f'{prefix}*' - flag_pairs = ( ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items() ) - return tuple( - element for pair in flag_pairs for element in pair - ) + flags.make_match_archives_flags( - storage_config.get('match_archives'), - storage_config.get('archive_name_format'), - local_borg_version, + return tuple(element for pair in flag_pairs for element in pair) + ( + ( + ('--match-archives', f'sh:{prefix}*') + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) + else ('--glob-archives', f'{prefix}*') + ) + if prefix + else ( + flags.make_match_archives_flags( + storage_config.get('match_archives'), + storage_config.get('archive_name_format'), + local_borg_version, + ) + ) ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 37731aca..128bdc0a 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -69,6 +69,24 @@ def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives() assert result == expected +def test_make_prune_flags_prefers_prefix_to_archive_name_format(): + storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 + retention_config = OrderedDict((('keep_daily', 1), ('prefix', 'bar-'))) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').never() + + result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') + + expected = ( + '--keep-daily', + '1', + '--match-archives', + 'sh:bar-*', # noqa: FS003 + ) + + assert result == expected + + def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 retention_config = OrderedDict((('keep_daily', 1), ('prefix', None))) From 0fbdf8d860ec47257f23621397396ccfa563927a Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 6 Apr 2023 09:31:24 +0530 Subject: [PATCH 055/344] feat: add logfile name to hook context for interpolation --- borgmatic/commands/borgmatic.py | 6 ++++++ tests/unit/commands/test_borgmatic.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 22aba0b1..b6d3c4bb 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -269,6 +269,12 @@ def run_actions( 'repositories': ','.join([repo['path'] for repo in location['repositories']]), } + try: + log_file = global_arguments.log_file + hook_context['log_file'] = log_file + except AttributeError: + pass + command.execute_hook( hooks.get('before_actions'), hooks.get('umask'), diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 11e879df..44a08590 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -422,6 +422,26 @@ def test_run_actions_runs_rcreate(): ) ) +def test_run_actions_adds_log_file_to_hook_context(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.rcreate).should_receive('run_rcreate').once() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), + hooks={}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) def test_run_actions_runs_transfer(): flexmock(module).should_receive('add_custom_log_levels') From 091d60c226f1d66f98f21252d6b072b62080b57c Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 6 Apr 2023 12:36:10 +0530 Subject: [PATCH 056/344] refactor and improve tests --- borgmatic/commands/borgmatic.py | 6 +-- tests/unit/commands/test_borgmatic.py | 53 +++++++++++++++++---------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b6d3c4bb..a4de94ba 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -269,11 +269,7 @@ def run_actions( 'repositories': ','.join([repo['path'] for repo in location['repositories']]), } - try: - log_file = global_arguments.log_file - hook_context['log_file'] = log_file - except AttributeError: - pass + hook_context['log_file'] = global_arguments.log_file if global_arguments.log_file else '' command.execute_hook( hooks.get('before_actions'), diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 44a08590..63501853 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -408,7 +408,7 @@ def test_run_actions_runs_rcreate(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'rcreate': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -425,11 +425,24 @@ def test_run_actions_runs_rcreate(): def test_run_actions_adds_log_file_to_hook_context(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') - flexmock(borgmatic.actions.rcreate).should_receive('run_rcreate').once() + flexmock(borgmatic.actions.create).should_receive('run_create').with_args( + config_filename=flexmock(), + repository={'path': 'repo'}, + location={'repositories': []}, + storage=flexmock(), + hooks={}, + hook_context={'log_file': 'foo'}, + local_borg_version=flexmock(), + create_arguments=flexmock(), + global_arguments=flexmock(dry_run=False, log_file='foo'), + dry_run_label='', + local_path=flexmock(), + remote_path=flexmock(), + ).once() tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -450,7 +463,7 @@ def test_run_actions_runs_transfer(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'transfer': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'transfer': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -473,7 +486,7 @@ def test_run_actions_runs_create(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'create': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -496,7 +509,7 @@ def test_run_actions_runs_prune(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'prune': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'prune': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -518,7 +531,7 @@ def test_run_actions_runs_compact(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'compact': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'compact': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -541,7 +554,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'check': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -564,7 +577,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'check': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -586,7 +599,7 @@ def test_run_actions_runs_extract(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'extract': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'extract': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -608,7 +621,7 @@ def test_run_actions_runs_export_tar(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'export-tar': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'export-tar': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -630,7 +643,7 @@ def test_run_actions_runs_mount(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'mount': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'mount': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -652,7 +665,7 @@ def test_run_actions_runs_restore(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'restore': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'restore': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -675,7 +688,7 @@ def test_run_actions_runs_rlist(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'rlist': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rlist': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -699,7 +712,7 @@ def test_run_actions_runs_list(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'list': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'list': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -723,7 +736,7 @@ def test_run_actions_runs_rinfo(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'rinfo': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rinfo': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -747,7 +760,7 @@ def test_run_actions_runs_info(): result = tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'info': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'info': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -770,7 +783,7 @@ def test_run_actions_runs_break_lock(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'break-lock': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'break-lock': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -792,7 +805,7 @@ def test_run_actions_runs_borg(): tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False), 'borg': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'borg': flexmock()}, config_filename=flexmock(), location={'repositories': []}, storage=flexmock(), @@ -816,7 +829,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): tuple( module.run_actions( arguments={ - 'global': flexmock(dry_run=False), + 'global': flexmock(dry_run=False, log_file='foo'), 'borg': flexmock(), 'restore': flexmock(), }, From 16d7131fb75b2aef9bd02c0af47607c86c209572 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 7 Apr 2023 01:00:38 +0530 Subject: [PATCH 057/344] refactor tests --- borgmatic/commands/borgmatic.py | 3 +-- tests/unit/commands/test_borgmatic.py | 24 ++++++++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a4de94ba..5d56accb 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -267,10 +267,9 @@ def run_actions( 'repository': repository_path, # Deprecated: For backwards compatibility with borgmatic < 1.6.0. 'repositories': ','.join([repo['path'] for repo in location['repositories']]), + 'log_file': global_arguments.log_file if global_arguments.log_file else '', } - hook_context['log_file'] = global_arguments.log_file if global_arguments.log_file else '' - command.execute_hook( hooks.get('before_actions'), hooks.get('umask'), diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 63501853..56f2332c 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -422,25 +422,27 @@ def test_run_actions_runs_rcreate(): ) ) + def test_run_actions_adds_log_file_to_hook_context(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') + expected = flexmock() flexmock(borgmatic.actions.create).should_receive('run_create').with_args( - config_filename=flexmock(), + config_filename=object, repository={'path': 'repo'}, location={'repositories': []}, - storage=flexmock(), + storage=object, hooks={}, - hook_context={'log_file': 'foo'}, - local_borg_version=flexmock(), - create_arguments=flexmock(), - global_arguments=flexmock(dry_run=False, log_file='foo'), + hook_context={'repository': 'repo', 'repositories': '', 'log_file': 'foo'}, + local_borg_version=object, + create_arguments=object, + global_arguments=object, dry_run_label='', - local_path=flexmock(), - remote_path=flexmock(), - ).once() + local_path=object, + remote_path=object, + ).once().and_return(expected) - tuple( + result = tuple( module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), @@ -455,6 +457,8 @@ def test_run_actions_adds_log_file_to_hook_context(): repository={'path': 'repo'}, ) ) + assert result == (expected,) + def test_run_actions_runs_transfer(): flexmock(module).should_receive('add_custom_log_levels') From 4cd7556a34f1cff6c8c87047f18e6b6872567ab9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Apr 2023 13:58:37 -0700 Subject: [PATCH 058/344] Add "log_file" command hook context to NEWS and docs (#413). --- NEWS | 2 ++ docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/NEWS b/NEWS index f80169a8..1c5bb6f4 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.7.12.dev0 + * #413: Add "log_file" context to command hooks. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the "--match-archives" flag to correctly override the "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index c0bcfb89..426e1a85 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -66,6 +66,9 @@ variables you can use here: * `configuration_filename`: borgmatic configuration filename in which the hook was defined + * `log_file` + New in version 1.7.12: + path of the borgmatic log file, only set when the `--log-file` flag is used * `repository`: path of the current repository as configured in the current borgmatic configuration file From cc60a71210065767ffd598b57f79767be637fb05 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Apr 2023 14:12:12 -0700 Subject: [PATCH 059/344] Clarify "log_file" NEWS (#413). --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 1c5bb6f4..8c8f0ffc 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.7.12.dev0 - * #413: Add "log_file" context to command hooks. See the documentation for more information: + * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. + See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the "--match-archives" flag to correctly override the "match_archives" configuration option for From d6ef0df50d8b011de4d49ebc27fbe9962be0c893 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Apr 2023 10:01:08 -0700 Subject: [PATCH 060/344] Mention #670 being fixed in NEWS. --- NEWS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 8c8f0ffc..cd851826 100644 --- a/NEWS +++ b/NEWS @@ -2,9 +2,9 @@ * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ - * #666: Fix error when running the "info" action with the "--match-archives" flag. Also fix the - "--match-archives" flag to correctly override the "match_archives" configuration option for - the "transfer", "list", "rlist", and "info" actions. + * #666, #670: Fix error when running the "info" action with the "--match-archives" or "--archive" + flags. Also fix the "--match-archives"/"--archive" flags to correctly override the + "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set. From 31a2ac914aa87d7d00c7b18ea04ec797bc96a261 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Apr 2023 14:26:54 -0700 Subject: [PATCH 061/344] Add optional support for running end-to-end tests and building documentation with rootless Podman instead of Docker. --- NEWS | 2 ++ docs/Dockerfile | 6 ++--- docs/how-to/develop-on-borgmatic.md | 36 ++++++++++++++++++++++++++++ scripts/run-end-to-end-dev-tests | 6 +++++ tests/end-to-end/docker-compose.yaml | 8 +++---- 5 files changed, 51 insertions(+), 7 deletions(-) diff --git a/NEWS b/NEWS index cd851826..81ebcf31 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,8 @@ "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set. + * Add optional support for running end-to-end tests and building documentation with rootless Podman + instead of Docker. 1.7.11 * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives diff --git a/docs/Dockerfile b/docs/Dockerfile index dcda44f3..b612596e 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.17.1 as borgmatic +FROM docker.io/alpine:3.17.1 as borgmatic COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib @@ -8,7 +8,7 @@ RUN borgmatic --help > /command-line.txt \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic "$action" --help >> /command-line.txt; done -FROM node:19.5.0-alpine as html +FROM docker.io/node:19.5.0-alpine as html ARG ENVIRONMENT=production @@ -28,7 +28,7 @@ COPY . /source RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \ && mv /output/docs/index.html /output/index.html -FROM nginx:1.22.1-alpine +FROM docker.io/nginx:1.22.1-alpine COPY --from=html /output /usr/share/nginx/html COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 6699bdf3..027a8145 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -101,6 +101,30 @@ scripts/run-end-to-end-dev-tests Note that this scripts assumes you have permission to run Docker. If you don't, then you may need to run with `sudo`. + +#### Podman + +New in version 1.7.12 +borgmatic's end-to-end tests optionally support using +[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md) +[Podman](https://podman.io/) instead of Docker. + +Setting up Podman is outside the scope of this documentation, but here are +some key points to double-check: + + * Install Podman along with `podman-docker` and your desired networking + support. + * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the + non-root user who will run tests. + * Create a non-root Podman socket for that user: + ```bash + systemctl --user enable --now podman.socket + ``` + +Then you'll be able to run end-to-end tests as per normal, and the test script +will automatically use your non-root Podman socket instead of a Docker socket. + + ## Code style Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply @@ -149,3 +173,15 @@ http://localhost:8080 to view the documentation with your changes. To close the documentation server, ctrl-C the script. Note that it does not currently auto-reload, so you'll need to stop it and re-run it for any additional documentation changes to take effect. + + +#### Podman + +New in version 1.7.12 +borgmatic's developer build for documentation optionally supports using +[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md) +[Podman](https://podman.io/) instead of Docker. + +Setting up Podman is outside the scope of this documentation. But once you +install `podman-docker`, then `scripts/dev-docs` should automatically use +Podman instead of Docker. diff --git a/scripts/run-end-to-end-dev-tests b/scripts/run-end-to-end-dev-tests index 575de00d..032967fd 100755 --- a/scripts/run-end-to-end-dev-tests +++ b/scripts/run-end-to-end-dev-tests @@ -10,5 +10,11 @@ set -e +USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock + +if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then + export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH" +fi + docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \ --renew-anon-volumes --abort-on-container-exit diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 80f12e9a..e6b0bf2b 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -1,22 +1,22 @@ version: '3' services: postgresql: - image: postgres:13.1-alpine + image: docker.io/postgres:13.1-alpine environment: POSTGRES_PASSWORD: test POSTGRES_DB: test mysql: - image: mariadb:10.5 + image: docker.io/mariadb:10.5 environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test mongodb: - image: mongo:5.0.5 + image: docker.io/mongo:5.0.5 environment: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: test tests: - image: alpine:3.13 + image: docker.io/alpine:3.13 environment: TEST_CONTAINER: true volumes: From 4c0e2cab78f2b13823a8f44f957911422770b960 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Apr 2023 10:49:09 -0700 Subject: [PATCH 062/344] View the results of configuration file merging via "validate-borgmatic-config --show" flag (#673). --- NEWS | 3 +++ borgmatic/commands/validate_config.py | 26 +++++++++++++++---- docs/how-to/make-per-application-backups.md | 17 ++++++++++++ docs/how-to/set-up-backups.md | 3 +++ tests/end-to-end/test_validate_config.py | 14 ++++++++++ .../commands/test_validate_config.py | 9 +++++++ 6 files changed, 67 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 81ebcf31..c9869b90 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,9 @@ "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set. + * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag. + See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes * Add optional support for running end-to-end tests and building documentation with rootless Podman instead of Docker. diff --git a/borgmatic/commands/validate_config.py b/borgmatic/commands/validate_config.py index 44c0082a..8aa8d321 100644 --- a/borgmatic/commands/validate_config.py +++ b/borgmatic/commands/validate_config.py @@ -2,6 +2,7 @@ import logging import sys from argparse import ArgumentParser +import borgmatic.config.generate from borgmatic.config import collect, validate logger = logging.getLogger(__name__) @@ -23,16 +24,22 @@ def parse_arguments(*arguments): default=config_paths, help=f'Configuration filenames or directories, defaults to: {config_paths}', ) + parser.add_argument( + '-s', + '--show', + action='store_true', + help='Show the validated configuration after all include merging has occurred', + ) return parser.parse_args(arguments) def main(): # pragma: no cover - args = parse_arguments(*sys.argv[1:]) + arguments = parse_arguments(*sys.argv[1:]) logging.basicConfig(level=logging.INFO, format='%(message)s') - config_filenames = tuple(collect.collect_config_filenames(args.config_paths)) + config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths)) if len(config_filenames) == 0: logger.critical('No files to validate found') sys.exit(1) @@ -40,13 +47,22 @@ def main(): # pragma: no cover found_issues = False for config_filename in config_filenames: try: - validate.parse_configuration(config_filename, validate.schema_filename()) + config, parse_logs = validate.parse_configuration( + config_filename, validate.schema_filename() + ) except (ValueError, OSError, validate.Validation_error) as error: logging.critical(f'{config_filename}: Error parsing configuration file') logging.critical(error) found_issues = True + else: + for log in parse_logs: + logger.handle(log) + + if arguments.show: + print('---') + print(borgmatic.config.generate.render_configuration(config)) if found_issues: sys.exit(1) - else: - logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}") + + logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}") diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 99f665bc..ab293472 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -276,6 +276,23 @@ include, the local file's option takes precedence. list values are appended together. +## Debugging includes + +New in version 1.7.12 If you'd +like to see what the loaded configuration looks like after includes get merged +in, run `validate-borgmatic-config` on your configuration file: + +```bash +sudo validate-borgmatic-config --show +``` + +You'll need to specify your configuration file with `--config` if it's not in +a default location. + +This will output the merged configuration as borgmatic sees it, which can be +helpful for understanding how your includes work in practice. + + ## Configuration overrides In more complex multi-application setups, you may want to override particular diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 01d79ff8..917eb437 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -180,6 +180,9 @@ following command is available for that: sudo validate-borgmatic-config ``` +You'll need to specify your configuration file with `--config` if it's not in +a default location. + This command's exit status (`$?` in Bash) is zero when configuration is valid and non-zero otherwise. diff --git a/tests/end-to-end/test_validate_config.py b/tests/end-to-end/test_validate_config.py index d41464e6..54465033 100644 --- a/tests/end-to-end/test_validate_config.py +++ b/tests/end-to-end/test_validate_config.py @@ -1,5 +1,6 @@ import os import subprocess +import sys import tempfile @@ -26,3 +27,16 @@ def test_validate_config_command_with_invalid_configuration_fails(): exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' ')) assert exit_code == 1 + + +def test_validate_config_command_with_show_flag_displays_configuration(): + with tempfile.TemporaryDirectory() as temporary_directory: + config_path = os.path.join(temporary_directory, 'test.yaml') + + subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + output = subprocess.check_output( + f'validate-borgmatic-config --config {config_path} --show'.split(' ') + ).decode(sys.stdout.encoding) + + assert 'location:' in output + assert 'repositories:' in output diff --git a/tests/integration/commands/test_validate_config.py b/tests/integration/commands/test_validate_config.py index cb0144d2..78887e71 100644 --- a/tests/integration/commands/test_validate_config.py +++ b/tests/integration/commands/test_validate_config.py @@ -18,3 +18,12 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list(): parser = module.parse_arguments('--config', 'myconfig', 'otherconfig') assert parser.config_paths == ['myconfig', 'otherconfig'] + + +def test_parse_arguments_supports_show_flag(): + config_paths = ['default'] + flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) + + parser = module.parse_arguments('--config', 'myconfig', '--show') + + assert parser.show From 264cebd2b10cd13c3782d639bba32e07f556c2be Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 11 Apr 2023 23:19:49 +0530 Subject: [PATCH 063/344] complete psql multi schema backup --- borgmatic/actions/restore.py | 9 ++------- borgmatic/commands/arguments.py | 2 +- borgmatic/hooks/postgresql.py | 7 +++++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index fa7b04b7..1e8d175d 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -68,15 +68,12 @@ def restore_single_database( archive_name, hook_name, database, - schemas ): # pragma: no cover ''' Given (among other things) an archive name, a database hook name, and a configured database configuration dict, restore that database from the archive. ''' logger.info(f'{repository}: Restoring database {database["name"]}') - if schemas: - database['schemas'] = schemas dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', @@ -316,8 +313,7 @@ def run_restore( remote_path, archive_name, found_hook_name or hook_name, - found_database, - schemas = restore_arguments.schemas, + dict(found_database, **{'schemas': restore_arguments.schemas}), ) # For any database that weren't found via exact matches in the hooks configuration, try to @@ -346,8 +342,7 @@ def run_restore( remote_path, archive_name, found_hook_name or hook_name, - database, - schemas = restore_arguments.schemas, + dict(database, **{'schemas': restore_arguments.schemas}), ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 7f93910e..ebd7b8eb 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -634,7 +634,7 @@ def make_parsers(): metavar='NAME', nargs='+', dest='schemas', - help="Names of schemas to restore from the database, defaults to all schemas." + help="Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases", ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 5d951f6d..d296fa1c 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -213,7 +213,6 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ('--command', 'ANALYZE') ) pg_restore_command = database.get('pg_restore_command') or 'pg_restore' - backup_schemas = ', '.join(database['schemas']) if 'schemas' in database else None restore_command = ( (psql_command if all_databases else pg_restore_command, '--no-password') + ( @@ -224,10 +223,14 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) - + (('--schema', backup_schemas) if backup_schemas else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) ) + + if database['schemas']: + for schema in database['schemas']: + restore_command += ('--schema', schema) + extra_environment = make_extra_environment(database) logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") From 2fea429d785ba9902057472c2d3afc2cdb7ffa79 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 12 Apr 2023 09:34:19 +0530 Subject: [PATCH 064/344] collection restore for mongodb --- borgmatic/hooks/mongodb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index be5f656b..781e5f21 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -161,4 +161,7 @@ def build_restore_command(extract_process, database, dump_filename): command.extend(('--authenticationDatabase', database['authentication_database'])) if 'restore_options' in database: command.extend(database['restore_options'].split(' ')) + if database['schemas']: + for schema in database['schemas']: + command.extend(('--nsInclude', schema)) return command From 1ea4433aa98c6a15b6ba223f2835b900e54b5c55 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Apr 2023 21:49:10 -0700 Subject: [PATCH 065/344] Selectively shallow merge certain mappings or sequences when including configuration files (#672). --- NEWS | 3 + borgmatic/config/load.py | 79 +++-- docs/how-to/make-per-application-backups.md | 59 ++++ tests/integration/config/test_load.py | 303 +++++++++++++++++--- 4 files changed, 374 insertions(+), 70 deletions(-) diff --git a/NEWS b/NEWS index c9869b90..6d4f5448 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,9 @@ "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions. * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix" options set. + * #672: Selectively shallow merge certain mappings or sequences when including configuration files. + See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index f3c45d52..e9caeb4c 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -38,6 +38,24 @@ def include_configuration(loader, filename_node, include_directory): return load_configuration(include_filename) +def retain_node_error(loader, node): + ''' + Given a ruamel.yaml.loader.Loader and a YAML node, raise an error. + + Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was + used in a configuration file without a merge. In configuration files with a merge, mapping and + sequence nodes with "!retain" tags are handled by deep_merge_nodes() below. + + Also raise ValueError if a scalar node is given, as "!retain" is not supported on scalar nodes. + ''' + if isinstance(node, (ruamel.yaml.nodes.MappingNode, ruamel.yaml.nodes.SequenceNode)): + raise ValueError( + 'The !retain tag may only be used within a configuration file containing a merged !include tag.' + ) + + raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.') + + class Include_constructor(ruamel.yaml.SafeConstructor): ''' A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including @@ -50,6 +68,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor): '!include', functools.partial(include_configuration, include_directory=include_directory), ) + self.add_constructor('!retain', retain_node_error) def flatten_mapping(self, node): ''' @@ -176,6 +195,8 @@ def deep_merge_nodes(nodes): ), ] + If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged. + The purpose of deep merging like this is to support, for instance, merging one borgmatic configuration file into another for reuse, such that a configuration section ("retention", etc.) does not completely replace the corresponding section in a merged file. @@ -198,32 +219,42 @@ def deep_merge_nodes(nodes): # If we're dealing with MappingNodes, recurse and merge its values as well. if isinstance(b_value, ruamel.yaml.nodes.MappingNode): - replaced_nodes[(b_key, b_value)] = ( - b_key, - ruamel.yaml.nodes.MappingNode( - tag=b_value.tag, - value=deep_merge_nodes(a_value.value + b_value.value), - start_mark=b_value.start_mark, - end_mark=b_value.end_mark, - flow_style=b_value.flow_style, - comment=b_value.comment, - anchor=b_value.anchor, - ), - ) + # A "!retain" tag says to skip deep merging for this node. Replace the tag so + # downstream schema validation doesn't break on our application-specific tag. + if b_value.tag == '!retain': + b_value.tag = 'tag:yaml.org,2002:map' + else: + replaced_nodes[(b_key, b_value)] = ( + b_key, + ruamel.yaml.nodes.MappingNode( + tag=b_value.tag, + value=deep_merge_nodes(a_value.value + b_value.value), + start_mark=b_value.start_mark, + end_mark=b_value.end_mark, + flow_style=b_value.flow_style, + comment=b_value.comment, + anchor=b_value.anchor, + ), + ) # If we're dealing with SequenceNodes, merge by appending one sequence to the other. elif isinstance(b_value, ruamel.yaml.nodes.SequenceNode): - replaced_nodes[(b_key, b_value)] = ( - b_key, - ruamel.yaml.nodes.SequenceNode( - tag=b_value.tag, - value=a_value.value + b_value.value, - start_mark=b_value.start_mark, - end_mark=b_value.end_mark, - flow_style=b_value.flow_style, - comment=b_value.comment, - anchor=b_value.anchor, - ), - ) + # A "!retain" tag says to skip deep merging for this node. Replace the tag so + # downstream schema validation doesn't break on our application-specific tag. + if b_value.tag == '!retain': + b_value.tag = 'tag:yaml.org,2002:seq' + else: + replaced_nodes[(b_key, b_value)] = ( + b_key, + ruamel.yaml.nodes.SequenceNode( + tag=b_value.tag, + value=a_value.value + b_value.value, + start_mark=b_value.start_mark, + end_mark=b_value.end_mark, + flow_style=b_value.flow_style, + comment=b_value.comment, + anchor=b_value.anchor, + ), + ) return [ replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index ab293472..ddaa37d2 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -276,6 +276,65 @@ include, the local file's option takes precedence. list values are appended together. +### Shallow merge + +Even though deep merging is generally pretty handy for included files, +sometimes you want specific sections in the local file to take precedence over +included sections—without any merging occuring for them. + +New in version 1.7.12 That's +where the `!retain` tag comes in. Whenever you're merging an included file +into your configuration file, you can optionally add the `!retain` tag to +particular local mappings or sequences to retain the local values and ignore +included values. + +For instance, start with this configuration file containing the `!retain` tag +on the `retention` mapping: + +```yaml +<<: !include /etc/borgmatic/common.yaml + +location: + repositories: + - repo.borg + +retention: !retain + keep_daily: 5 +``` + +And `common.yaml` like this: + +```yaml +location: + repositories: + - common.borg + +retention: + keep_hourly: 24 + keep_daily: 7 +``` + +Once this include gets merged in, the resulting configuration will have a +`keep_daily` value of `5` and nothing else in the `retention` section. That's +because the `!retain` tag says to retain the local version of `retention` and +ignore any values coming in from the include. But because the `repositories` +sequence doesn't have a `!retain` tag, that sequence still gets merged +together to contain both `common.borg` and `repo.borg`. + +The `!retain` tag can only be placed on mapping and sequence nodes, and it +goes right after the name of the option (and its colon) on the same line. The +effects of `!retain` are recursive, meaning that if you place a `!retain` tag +on a top-level mapping, even deeply nested values within it will not be +merged. + +Additionally, the `!retain` tag only works in a configuration file that also +performs a merge include with `<<: !include`. It doesn't make sense within, +for instance, an included configuration file itself (unless it in turn +performs its own merge include). That's because `!retain` only applies to the +file doing the include; it doesn't work in reverse or propagate through +includes. + + ## Debugging includes New in version 1.7.12 If you'd diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index 3382d734..ef9be0f8 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -2,7 +2,6 @@ import io import sys import pytest -import ruamel.yaml from flexmock import flexmock from borgmatic.config import load as module @@ -150,6 +149,99 @@ def test_load_configuration_merges_include(): assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'} +def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + foo: bar + baz: quux + + other: + a: b + c: d + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: !retain + foo: override + + other: + a: override + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + assert module.load_configuration('config.yaml') == { + 'stuff': {'foo': 'override'}, + 'other': {'a': 'override', 'c': 'd'}, + } + + +def test_load_configuration_with_retain_tag_but_without_merge_include_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: !retain + foo: bar + baz: quux + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + foo: override + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + assert module.load_configuration('config.yaml') + + +def test_load_configuration_with_retain_tag_on_scalar_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + foo: bar + baz: quux + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + foo: !retain override + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + assert module.load_configuration('config.yaml') + + def test_load_configuration_does_not_merge_include_list(): builtins = flexmock(sys.modules['builtins']) flexmock(module.os).should_receive('getcwd').and_return('/tmp') @@ -173,42 +265,59 @@ def test_load_configuration_does_not_merge_include_list(): config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) - with pytest.raises(ruamel.yaml.error.YAMLError): + with pytest.raises(module.ruamel.yaml.error.YAMLError): assert module.load_configuration('config.yaml') +@pytest.mark.parametrize( + 'node_class', + ( + module.ruamel.yaml.nodes.MappingNode, + module.ruamel.yaml.nodes.SequenceNode, + module.ruamel.yaml.nodes.ScalarNode, + ), +) +def test_retain_node_error_raises(node_class): + with pytest.raises(ValueError): + module.retain_node_error( + loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock()) + ) + + def test_deep_merge_nodes_replaces_colliding_scalar_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_hourly' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='24' + ), ), ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_daily' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), ), ], ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_daily' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), ), ], ), @@ -230,35 +339,39 @@ def test_deep_merge_nodes_replaces_colliding_scalar_values(): def test_deep_merge_nodes_keeps_non_colliding_scalar_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_hourly' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='24' + ), ), ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_daily' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), ), ], ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='keep_minutely' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='10'), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='10' + ), ), ], ), @@ -282,28 +395,28 @@ def test_deep_merge_nodes_keeps_non_colliding_scalar_values(): def test_deep_merge_nodes_keeps_deeply_nested_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='lock_wait' ), - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), ), ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='extra_borg_options' ), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='init' ), - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='--init-option' ), ), @@ -314,22 +427,22 @@ def test_deep_merge_nodes_keeps_deeply_nested_values(): ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='extra_borg_options' ), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='prune' ), - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='--prune-option' ), ), @@ -361,32 +474,32 @@ def test_deep_merge_nodes_keeps_deeply_nested_values(): def test_deep_merge_nodes_appends_colliding_sequence_values(): node_values = [ ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='before_backup' ), - ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:int', value=['echo 1', 'echo 2'] + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] ), ), ], ), ), ( - ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), - ruamel.yaml.nodes.MappingNode( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( tag='tag:yaml.org,2002:map', value=[ ( - ruamel.yaml.nodes.ScalarNode( + module.ruamel.yaml.nodes.ScalarNode( tag='tag:yaml.org,2002:str', value='before_backup' ), - ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:int', value=['echo 3', 'echo 4'] + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', value=['echo 3', 'echo 4'] ), ), ], @@ -402,3 +515,101 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): assert len(options) == 1 assert options[0][0].value == 'before_backup' assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] + + +def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='keep_hourly' + ), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:int', value='24' + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='keep_daily' + ), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), + module.ruamel.yaml.nodes.MappingNode( + tag='!retain', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='keep_daily' + ), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'), + ), + ], + ), + ), + ] + + result = module.deep_merge_nodes(node_values) + assert len(result) == 1 + (section_key, section_value) = result[0] + assert section_key.value == 'retention' + assert section_value.tag == 'tag:yaml.org,2002:map' + options = section_value.value + assert len(options) == 1 + assert options[0][0].value == 'keep_daily' + assert options[0][1].value == '5' + + +def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] + ), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='!retain', value=['echo 3', 'echo 4'] + ), + ), + ], + ), + ), + ] + + result = module.deep_merge_nodes(node_values) + assert len(result) == 1 + (section_key, section_value) = result[0] + assert section_key.value == 'hooks' + options = section_value.value + assert len(options) == 1 + assert options[0][0].value == 'before_backup' + assert options[0][1].tag == 'tag:yaml.org,2002:seq' + assert options[0][1].value == ['echo 3', 'echo 4'] From ea9213cb0384180455f1f21c1d416db159c90e72 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Apr 2023 22:12:57 -0700 Subject: [PATCH 066/344] Spelling. --- docs/how-to/make-per-application-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index ddaa37d2..9a3bfa33 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -280,7 +280,7 @@ list values are appended together. Even though deep merging is generally pretty handy for included files, sometimes you want specific sections in the local file to take precedence over -included sections—without any merging occuring for them. +included sections—without any merging occurring for them. New in version 1.7.12 That's where the `!retain` tag comes in. Whenever you're merging an included file From 08843d51d9946043dc47c717a9786b277952fb2c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Apr 2023 10:30:23 -0700 Subject: [PATCH 067/344] Replace "sequence" with "list" in docs for consistency. --- docs/how-to/make-per-application-backups.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 9a3bfa33..887bc431 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -285,7 +285,7 @@ included sections—without any merging occurring for them. New in version 1.7.12 That's where the `!retain` tag comes in. Whenever you're merging an included file into your configuration file, you can optionally add the `!retain` tag to -particular local mappings or sequences to retain the local values and ignore +particular local mappings or lists to retain the local values and ignore included values. For instance, start with this configuration file containing the `!retain` tag @@ -318,14 +318,13 @@ Once this include gets merged in, the resulting configuration will have a `keep_daily` value of `5` and nothing else in the `retention` section. That's because the `!retain` tag says to retain the local version of `retention` and ignore any values coming in from the include. But because the `repositories` -sequence doesn't have a `!retain` tag, that sequence still gets merged -together to contain both `common.borg` and `repo.borg`. +list doesn't have a `!retain` tag, it still gets merged together to contain +both `common.borg` and `repo.borg`. -The `!retain` tag can only be placed on mapping and sequence nodes, and it -goes right after the name of the option (and its colon) on the same line. The -effects of `!retain` are recursive, meaning that if you place a `!retain` tag -on a top-level mapping, even deeply nested values within it will not be -merged. +The `!retain` tag can only be placed on mappings and lists, and it goes right +after the name of the option (and its colon) on the same line. The effects of +`!retain` are recursive, meaning that if you place a `!retain` tag on a +top-level mapping, even deeply nested values within it will not be merged. Additionally, the `!retain` tag only works in a configuration file that also performs a merge include with `<<: !include`. It doesn't make sense within, From 4a94c2c9bff7e7e17db595086a92647e18a23d23 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 13 Apr 2023 14:39:36 -0700 Subject: [PATCH 068/344] Selectively omit list values when including configuration files (#672). --- NEWS | 3 + borgmatic/config/load.py | 32 ++- docs/how-to/make-per-application-backups.md | 60 ++++- tests/integration/config/test_load.py | 285 +++++++++++++++++++- 4 files changed, 362 insertions(+), 18 deletions(-) diff --git a/NEWS b/NEWS index 6d4f5448..a39e84d2 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,9 @@ * #672: Selectively shallow merge certain mappings or sequences when including configuration files. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge + * #672: Selectively omit list values when including configuration files. See the documentation for + more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#list-merge * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index e9caeb4c..068e661f 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -38,9 +38,9 @@ def include_configuration(loader, filename_node, include_directory): return load_configuration(include_filename) -def retain_node_error(loader, node): +def raise_retain_node_error(loader, node): ''' - Given a ruamel.yaml.loader.Loader and a YAML node, raise an error. + Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!retain" usage. Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was used in a configuration file without a merge. In configuration files with a merge, mapping and @@ -56,6 +56,19 @@ def retain_node_error(loader, node): raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.') +def raise_omit_node_error(loader, node): + ''' + Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!omit" usage. + + Raise ValueError unconditionally, as an "!omit" node here indicates it was used in a + configuration file without a merge. In configuration files with a merge, nodes with "!omit" + tags are handled by deep_merge_nodes() below. + ''' + raise ValueError( + 'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.' + ) + + class Include_constructor(ruamel.yaml.SafeConstructor): ''' A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including @@ -68,7 +81,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor): '!include', functools.partial(include_configuration, include_directory=include_directory), ) - self.add_constructor('!retain', retain_node_error) + self.add_constructor('!retain', raise_retain_node_error) + self.add_constructor('!omit', raise_omit_node_error) def flatten_mapping(self, node): ''' @@ -134,6 +148,16 @@ def load_configuration(filename): return config +def filter_omitted_nodes(nodes): + ''' + Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a + value matching such nodes. + ''' + omitted_values = tuple(node.value for node in nodes if node.tag == '!omit') + + return [node for node in nodes if node.value not in omitted_values] + + DELETED_NODE = object() @@ -247,7 +271,7 @@ def deep_merge_nodes(nodes): b_key, ruamel.yaml.nodes.SequenceNode( tag=b_value.tag, - value=a_value.value + b_value.value, + value=filter_omitted_nodes(a_value.value + b_value.value), start_mark=b_value.start_mark, end_mark=b_value.end_mark, flow_style=b_value.flow_style, diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 887bc431..9ee93f44 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -272,9 +272,65 @@ Once this include gets merged in, the resulting configuration would have a When there's an option collision between the local file and the merged include, the local file's option takes precedence. + +#### List merge + New in version 1.6.1 Colliding list values are appended together. +New in version 1.7.12 If there +is a list value from an include that you *don't* want in your local +configuration file, you can omit it with an `!omit` tag. For instance: + +```yaml +<<: !include /etc/borgmatic/common.yaml + +location: + source_directories: + - !omit /home + - /var +``` + +And `common.yaml` like this: + +```yaml +location: + source_directories: + - /home + - /etc +``` + +Once this include gets merged in, the resulting configuration will have a +`source_directories` value of `/etc` and `/var`—with `/home` omitted. + +This feature currently only works on scalar (e.g. string or number) list items +and will not work elsewhere in a configuration file. Be sure to put the +`!omit` tag *before* the list item (after the dash). Putting `!omit` after the +list item will not work, as it gets interpreted as part of the string. Here's +an example of some things not to do: + +```yaml +<<: !include /etc/borgmatic/common.yaml + +location: + source_directories: + # Do not do this! It will not work. "!omit" belongs before "/home". + - /home !omit + + # Do not do this either! "!omit" only works on scalar list items. + repositories: !omit + # Also do not do this for the same reason! This is a list item, but it's + # not a scalar. + - !omit path: repo.borg +``` + +Additionally, the `!omit` tag only works in a configuration file that also +performs a merge include with `<<: !include`. It doesn't make sense within, +for instance, an included configuration file itself (unless it in turn +performs its own merge include). That's because `!omit` only applies to the +file doing the include; it doesn't work in reverse or propagate through +includes. + ### Shallow merge @@ -296,7 +352,7 @@ on the `retention` mapping: location: repositories: - - repo.borg + - path: repo.borg retention: !retain keep_daily: 5 @@ -307,7 +363,7 @@ And `common.yaml` like this: ```yaml location: repositories: - - common.borg + - path: common.borg retention: keep_hourly: 24 diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index ef9be0f8..028a6523 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -211,7 +211,7 @@ def test_load_configuration_with_retain_tag_but_without_merge_include_raises(): builtins.should_receive('open').with_args('config.yaml').and_return(config_file) with pytest.raises(ValueError): - assert module.load_configuration('config.yaml') + module.load_configuration('config.yaml') def test_load_configuration_with_retain_tag_on_scalar_raises(): @@ -239,7 +239,156 @@ def test_load_configuration_with_retain_tag_on_scalar_raises(): builtins.should_receive('open').with_args('config.yaml').and_return(config_file) with pytest.raises(ValueError): - assert module.load_configuration('config.yaml') + module.load_configuration('config.yaml') + + +def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - x + - !omit b + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']} + + +def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - x + - !omit q + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']} + + +def test_load_configuration_with_omit_tag_on_non_list_item_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: !omit + - x + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') + + +def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - foo: bar + baz: quux + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - !omit foo: bar + baz: quux + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') + + +def test_load_configuration_with_omit_tag_but_without_merge_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + include_file = io.StringIO( + ''' + stuff: + - a + - !omit b + - c + ''' + ) + include_file.name = 'include.yaml' + builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) + config_file = io.StringIO( + ''' + stuff: + - x + - y + <<: !include include.yaml + ''' + ) + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + + with pytest.raises(ValueError): + module.load_configuration('config.yaml') def test_load_configuration_does_not_merge_include_list(): @@ -277,13 +426,33 @@ def test_load_configuration_does_not_merge_include_list(): module.ruamel.yaml.nodes.ScalarNode, ), ) -def test_retain_node_error_raises(node_class): +def test_raise_retain_node_error_raises(node_class): with pytest.raises(ValueError): - module.retain_node_error( + module.raise_retain_node_error( loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock()) ) +def test_raise_omit_node_error_raises(): + with pytest.raises(ValueError): + module.raise_omit_node_error(loader=flexmock(), node=flexmock()) + + +def test_filter_omitted_nodes(): + nodes = [ + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'), + module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'), + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'), + ] + + result = module.filter_omitted_nodes(nodes) + + assert [item.value for item in result] == ['a', 'c', 'a', 'c'] + + def test_deep_merge_nodes_replaces_colliding_scalar_values(): node_values = [ ( @@ -483,7 +652,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): tag='tag:yaml.org,2002:str', value='before_backup' ), module.ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 1' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 2' + ), + ], ), ), ], @@ -499,7 +676,15 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): tag='tag:yaml.org,2002:str', value='before_backup' ), module.ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:seq', value=['echo 3', 'echo 4'] + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 4' + ), + ], ), ), ], @@ -514,10 +699,10 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): options = section_value.value assert len(options) == 1 assert options[0][0].value == 'before_backup' - assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] + assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] -def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain(): +def test_deep_merge_nodes_only_keeps_mapping_values_tagged_with_retain(): node_values = [ ( module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'), @@ -568,7 +753,7 @@ def test_deep_merge_nodes_keeps_mapping_values_tagged_with_retain(): assert options[0][1].value == '5' -def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): +def test_deep_merge_nodes_only_keeps_sequence_values_tagged_with_retain(): node_values = [ ( module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), @@ -580,7 +765,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): tag='tag:yaml.org,2002:str', value='before_backup' ), module.ruamel.yaml.nodes.SequenceNode( - tag='tag:yaml.org,2002:seq', value=['echo 1', 'echo 2'] + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 1' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 2' + ), + ], ), ), ], @@ -596,7 +789,15 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): tag='tag:yaml.org,2002:str', value='before_backup' ), module.ruamel.yaml.nodes.SequenceNode( - tag='!retain', value=['echo 3', 'echo 4'] + tag='!retain', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 4' + ), + ], ), ), ], @@ -612,4 +813,64 @@ def test_deep_merge_nodes_keeps_sequence_values_tagged_with_retain(): assert len(options) == 1 assert options[0][0].value == 'before_backup' assert options[0][1].tag == 'tag:yaml.org,2002:seq' - assert options[0][1].value == ['echo 3', 'echo 4'] + assert [item.value for item in options[0][1].value] == ['echo 3', 'echo 4'] + + +def test_deep_merge_nodes_skips_sequence_values_tagged_with_omit(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 1' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 2' + ), + ], + ), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode(tag='!omit', value='echo 2'), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + ], + ), + ), + ], + ), + ), + ] + + result = module.deep_merge_nodes(node_values) + assert len(result) == 1 + (section_key, section_value) = result[0] + assert section_key.value == 'hooks' + options = section_value.value + assert len(options) == 1 + assert options[0][0].value == 'before_backup' + assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 3'] From 613f6c602c19095538dbde585c275e40e1fe036d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 13 Apr 2023 15:12:19 -0700 Subject: [PATCH 069/344] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index a39e84d2..08003882 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.12.dev0 +1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ diff --git a/setup.py b/setup.py index 4b870eb4..92655fb9 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.12.dev0' +VERSION = '1.7.12' setup( From ebe5c5e83903685742a39ae63b779d7a64527cff Mon Sep 17 00:00:00 2001 From: polyzen Date: Fri, 14 Apr 2023 01:01:31 +0000 Subject: [PATCH 070/344] Fix "TypeError: 'module' object is not callable" in test_commands.py --- tests/integration/borg/test_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 49cd0e96..3f5f4445 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -1,6 +1,6 @@ import copy -import flexmock +from flexmock import flexmock import borgmatic.borg.info import borgmatic.borg.list @@ -105,4 +105,4 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): borgmatic.borg.info.display_archives_info( 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) - ) + ) \ No newline at end of file From 50b0a9ce383fcdd9847f03e1639bc263ab5b7c19 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 13 Apr 2023 19:13:50 -0700 Subject: [PATCH 071/344] Remove newline at end of file. --- tests/integration/borg/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 3f5f4445..1afb0e0f 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -105,4 +105,4 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): borgmatic.borg.info.display_archives_info( 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) - ) \ No newline at end of file + ) From 874fba76725ce76803b586525c57c59760404593 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 15:10:44 +0200 Subject: [PATCH 072/344] Fix PostgreSQL hook not using "psql_command" for list when dumping "all" --- borgmatic/hooks/postgresql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index d4799f5f..c0d4619b 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -59,8 +59,9 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): if dry_run: return () + psql_command = database.get('psql_command') or 'psql' list_command = ( - ('psql', '--list', '--no-password', '--csv', '--tuples-only') + (psql_command, '--list', '--no-password', '--csv', '--tuples-only') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) From 19a00371f5da3c501a8158b1dfb70ba40c09ac08 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 15:49:49 +0200 Subject: [PATCH 073/344] Run "psql" with "--no-psqlrc" Some settings in user's .psqlrc, e.g. "linestyle unicode", may break the CSV output. "--no-psqlrc" tells psql to not read startup file. This is not necessary for the analyze_command and restore_command (with all_databases), but it's generally recommended when running psql from a script. --- borgmatic/hooks/postgresql.py | 6 +++--- tests/unit/hooks/test_postgresql.py | 18 +++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index c0d4619b..1235d60e 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -61,7 +61,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): psql_command = database.get('psql_command') or 'psql' list_command = ( - (psql_command, '--list', '--no-password', '--csv', '--tuples-only') + (psql_command, '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -205,7 +205,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) psql_command = database.get('psql_command') or 'psql' analyze_command = ( - (psql_command, '--no-password', '--quiet') + (psql_command, '--no-password', '--no-psqlrc', '--quiet') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -219,7 +219,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ( ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) if not all_databases - else () + else ('--no-psqlrc',) ) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 349c04be..704d5a71 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -56,6 +56,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam 'psql', '--list', '--no-password', + '--no-psqlrc', '--csv', '--tuples-only', '--host', @@ -75,7 +76,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam def test_database_names_to_dump_with_all_and_format_lists_databases_with_username(): database = {'name': 'all', 'format': 'custom', 'username': 'postgres'} flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('psql', '--list', '--no-password', '--csv', '--tuples-only', '--username', 'postgres'), + ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--username', 'postgres'), extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') @@ -88,7 +89,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam def test_database_names_to_dump_with_all_and_format_lists_databases_with_options(): database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'} flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('psql', '--list', '--no-password', '--csv', '--tuples-only', '--harder'), + ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'), extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') @@ -433,7 +434,7 @@ def test_restore_database_dump_runs_pg_restore(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ('psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -487,6 +488,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): ( 'psql', '--no-password', + '--no-psqlrc', '--quiet', '--host', 'database.example.org', @@ -535,6 +537,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): ( 'psql', '--no-password', + '--no-psqlrc', '--quiet', '--username', 'postgres', @@ -580,6 +583,7 @@ def test_restore_database_dump_runs_pg_restore_with_options(): ( 'psql', '--no-password', + '--no-psqlrc', '--quiet', '--dbname', 'foo', @@ -603,14 +607,14 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( - ('psql', '--no-password'), + ('psql', '--no-password', '--no-psqlrc'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), + ('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -644,7 +648,7 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('special_psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ('special_psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -689,7 +693,7 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ('psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), extra_environment={'PGSSLMODE': 'disable'}, ).once() From 195024e505746ce9eb096422352762619b8fd429 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 16:11:42 +0200 Subject: [PATCH 074/344] Fix psql_command and pg_restore_command to accept command with arguments These commands are executed without `shell=True`, so the subprocess module treats e.g. "docker exec my_pg_container psql" as a single command (resulting in Errno 2 "No such file or directory") instead of a command with arguments. --- borgmatic/hooks/postgresql.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 1235d60e..fbe890a7 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -1,6 +1,7 @@ import csv import logging import os +import shlex from borgmatic.execute import ( execute_command, @@ -59,9 +60,10 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): if dry_run: return () - psql_command = database.get('psql_command') or 'psql' + psql_command = shlex.split(database.get('psql_command') or 'psql') list_command = ( - (psql_command, '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') + tuple(psql_command) + + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -203,9 +205,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, dump_filename = dump.make_database_dump_filename( make_dump_path(location_config), database['name'], database.get('hostname') ) - psql_command = database.get('psql_command') or 'psql' + psql_command = shlex.split(database.get('psql_command') or 'psql') analyze_command = ( - (psql_command, '--no-password', '--no-psqlrc', '--quiet') + tuple(psql_command) + + ('--no-password', '--no-psqlrc', '--quiet') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -213,9 +216,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + ('--command', 'ANALYZE') ) - pg_restore_command = database.get('pg_restore_command') or 'pg_restore' + pg_restore_command = shlex.split(database.get('pg_restore_command') or 'pg_restore') restore_command = ( - (psql_command if all_databases else pg_restore_command, '--no-password') + tuple(psql_command if all_databases else pg_restore_command) + + ('--no-password',) + ( ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) if not all_databases From dfccc1b94a57cf791bf4fe06b4353e20fa75c2ee Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 16:27:45 +0200 Subject: [PATCH 075/344] Exit on error when restoring all PostgreSQL databases "--set ON_ERROR_STOP=on" is equivalent to "--exit-on-error" in pg_restore. --- borgmatic/hooks/postgresql.py | 2 +- tests/unit/hooks/test_postgresql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index fbe890a7..b91b97b8 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -223,7 +223,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ( ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) if not all_databases - else ('--no-psqlrc',) + else ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') ) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 704d5a71..5f02978c 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -607,7 +607,7 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( - ('psql', '--no-password', '--no-psqlrc'), + ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, From f0f43174c6be21157d2d30955e14f04ddcead8a2 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 16:29:26 +0200 Subject: [PATCH 076/344] Swap if-else in restore_database_dump in postgresql hook for cleanliness --- borgmatic/hooks/postgresql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index b91b97b8..0f1a2df4 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -221,9 +221,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, tuple(psql_command if all_databases else pg_restore_command) + ('--no-password',) + ( - ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) - if not all_databases - else ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') + ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') + if all_databases + else ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) ) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) From 17f122bfe54a95e30cd186b552ae0efd295f7aa3 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Fri, 14 Apr 2023 17:16:42 +0200 Subject: [PATCH 077/344] Use psql instead of pg_restore when format is "plain" pg_restore: error: input file appears to be a text format dump. Please use psql. --- borgmatic/hooks/postgresql.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 0f1a2df4..a64bb534 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -216,15 +216,17 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + ('--command', 'ANALYZE') ) + use_psql_command = all_databases or database.get('format') == 'plain' pg_restore_command = shlex.split(database.get('pg_restore_command') or 'pg_restore') restore_command = ( - tuple(psql_command if all_databases else pg_restore_command) + tuple(psql_command if use_psql_command else pg_restore_command) + ('--no-password',) + ( ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') - if all_databases - else ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name']) + if use_psql_command + else ('--if-exists', '--exit-on-error', '--clean') ) + + (('--dbname', database['name']) if not all_databases else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) From f273e82d74d31f4706c3fcc721a5a97a185a539b Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 15 Apr 2023 02:57:51 +0530 Subject: [PATCH 078/344] add tests --- borgmatic/commands/arguments.py | 2 +- borgmatic/hooks/postgresql.py | 10 +++-- tests/unit/actions/test_restore.py | 32 ++++++++------ tests/unit/hooks/test_mongodb.py | 43 ++++++++++++++++--- tests/unit/hooks/test_postgresql.py | 66 +++++++++++++++++++++++++---- 5 files changed, 122 insertions(+), 31 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index ebd7b8eb..b89ca761 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -634,7 +634,7 @@ def make_parsers(): metavar='NAME', nargs='+', dest='schemas', - help="Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases", + help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases', ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index d296fa1c..a982209c 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -1,4 +1,5 @@ import csv +import itertools import logging import os @@ -225,12 +226,13 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + (('--username', database['username']) if 'username' in database else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) + + tuple( + itertools.chain.from_iterable(('--schema', schema) for schema in database['schemas']) + if database['schemas'] + else () + ) ) - if database['schemas']: - for schema in database['schemas']: - restore_command += ('--schema', schema) - extra_environment = make_extra_environment(database) logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 8255c87d..667a23c3 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -233,7 +233,7 @@ def test_run_restore_restores_each_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -246,7 +246,7 @@ def test_run_restore_restores_each_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).once() flexmock(module).should_receive('ensure_databases_found') @@ -256,7 +256,9 @@ def test_run_restore_restores_each_database(): storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), @@ -327,7 +329,7 @@ def test_run_restore_restores_database_configured_with_all_name(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -340,7 +342,7 @@ def test_run_restore_restores_database_configured_with_all_name(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).once() flexmock(module).should_receive('ensure_databases_found') @@ -350,7 +352,9 @@ def test_run_restore_restores_database_configured_with_all_name(): storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), @@ -399,7 +403,7 @@ def test_run_restore_skips_missing_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -412,7 +416,7 @@ def test_run_restore_skips_missing_database(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).never() flexmock(module).should_receive('ensure_databases_found') @@ -422,7 +426,9 @@ def test_run_restore_skips_missing_database(): storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), @@ -465,7 +471,7 @@ def test_run_restore_restores_databases_from_different_hooks(): remote_path=object, archive_name=object, hook_name='postgresql_databases', - database={'name': 'foo'}, + database={'name': 'foo', 'schemas': None}, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -478,7 +484,7 @@ def test_run_restore_restores_databases_from_different_hooks(): remote_path=object, archive_name=object, hook_name='mysql_databases', - database={'name': 'bar'}, + database={'name': 'bar', 'schemas': None}, ).once() flexmock(module).should_receive('ensure_databases_found') @@ -488,7 +494,9 @@ def test_run_restore_restores_databases_from_different_hooks(): storage=flexmock(), hooks=flexmock(), local_borg_version=flexmock(), - restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + restore_arguments=flexmock( + repository='repo', archive='archive', databases=flexmock(), schemas=None + ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), remote_path=flexmock(), diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 44e427f1..77b830b7 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -157,7 +157,7 @@ def test_dump_databases_runs_mongodumpall_for_all_databases(): def test_restore_database_dump_runs_mongorestore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -189,7 +189,9 @@ def test_restore_database_dump_errors_on_multiple_database_config(): def test_restore_database_dump_runs_mongorestore_with_hostname_and_port(): - database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + database_config = [ + {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -223,6 +225,7 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): 'username': 'mongo', 'password': 'trustsome1', 'authentication_database': 'admin', + 'schemas': None, } ] extract_process = flexmock(stdout=flexmock()) @@ -254,7 +257,7 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): def test_restore_database_dump_runs_mongorestore_with_options(): - database_config = [{'name': 'foo', 'restore_options': '--harder'}] + database_config = [{'name': 'foo', 'restore_options': '--harder', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -271,8 +274,36 @@ def test_restore_database_dump_runs_mongorestore_with_options(): ) +def test_restore_databases_dump_runs_mongorestore_with_schemas(): + database_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + [ + 'mongorestore', + '--archive', + '--drop', + '--db', + 'foo', + '--nsInclude', + 'bar', + '--nsInclude', + 'baz', + ], + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + ) + + def test_restore_database_dump_runs_psql_for_all_database_dump(): - database_config = [{'name': 'all'}] + database_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -290,7 +321,7 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_with_dry_run_skips_restore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') @@ -302,7 +333,7 @@ def test_restore_database_dump_with_dry_run_skips_restore(): def test_restore_database_dump_without_extract_process_restores_from_disk(): - database_config = [{'name': 'foo', 'format': 'directory'}] + database_config = [{'name': 'foo', 'format': 'directory', 'schemas': None}] flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path') diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 349c04be..70cff920 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -411,7 +411,7 @@ def test_dump_databases_runs_non_default_pg_dump(): def test_restore_database_dump_runs_pg_restore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -458,7 +458,9 @@ def test_restore_database_dump_errors_on_multiple_database_config(): def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): - database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + database_config = [ + {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -506,7 +508,9 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): def test_restore_database_dump_runs_pg_restore_with_username_and_password(): - database_config = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}] + database_config = [ + {'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return( @@ -553,7 +557,12 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): def test_restore_database_dump_runs_pg_restore_with_options(): database_config = [ - {'name': 'foo', 'restore_options': '--harder', 'analyze_options': '--smarter'} + { + 'name': 'foo', + 'restore_options': '--harder', + 'analyze_options': '--smarter', + 'schemas': None, + } ] extract_process = flexmock(stdout=flexmock()) @@ -596,7 +605,7 @@ def test_restore_database_dump_runs_pg_restore_with_options(): def test_restore_database_dump_runs_psql_for_all_database_dump(): - database_config = [{'name': 'all'}] + database_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -621,7 +630,12 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): database_config = [ - {'name': 'foo', 'pg_restore_command': 'special_pg_restore', 'psql_command': 'special_psql'} + { + 'name': 'foo', + 'pg_restore_command': 'special_pg_restore', + 'psql_command': 'special_psql', + 'schemas': None, + } ] extract_process = flexmock(stdout=flexmock()) @@ -654,7 +668,7 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): def test_restore_database_dump_with_dry_run_skips_restore(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') @@ -667,7 +681,7 @@ def test_restore_database_dump_with_dry_run_skips_restore(): def test_restore_database_dump_without_extract_process_restores_from_disk(): - database_config = [{'name': 'foo'}] + database_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') @@ -696,3 +710,39 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): module.restore_database_dump( database_config, 'test.yaml', {}, dry_run=False, extract_process=None ) + + +def test_restore_database_dump_with_schemas_restores_schemas(): + database_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] + + flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'pg_restore', + '--no-password', + '--if-exists', + '--exit-on-error', + '--clean', + '--dbname', + 'foo', + '/dump/path', + '--schema', + 'bar', + '--schema', + 'baz', + ), + processes=[], + output_log_level=logging.DEBUG, + input_file=None, + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=None + ) From 96d4a8ee45c71109cb5091b445c2d72cf8525acf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 16:33:06 -0700 Subject: [PATCH 079/344] Add "borgmatic restore --schema" flag to NEWS (#375). --- NEWS | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 08003882..9743d77d 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.7.13.dev0 + * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" + flag. + 1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. See the documentation for more information: diff --git a/setup.py b/setup.py index 92655fb9..f6fca3db 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.12' +VERSION = '1.7.13.dev0' setup( From 1c67db5d627a9f936297747e3c410e868f625450 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 16:40:58 -0700 Subject: [PATCH 080/344] Add documentation for "borgmatic restore --schema" (#375). --- NEWS | 3 ++- docs/how-to/backup-your-databases.md | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 9743d77d..4a27d074 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.7.13.dev0 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" - flag. + flag. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas 1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 0b46e4c1..91dba18f 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -324,6 +324,17 @@ includes any combined dump file named "all" and any other individual database dumps found in the archive. +### Restore particular schemas + +New in version 1.7.13 With +PostgreSQL and MongoDB, you can limit the restore to a single schema found +within the database dump: + +```bash +borgmatic restore --archive latest --database users --schema tentant1 +``` + + ### Limitations There are a few important limitations with borgmatic's current database From 5dbb71709c505c7a511209aeab1527af43591487 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 19:35:24 -0700 Subject: [PATCH 081/344] Upgrade test requirements and code style requirements. Auto-reformat code accordingly. --- borgmatic/actions/borg.py | 7 +- borgmatic/actions/break_lock.py | 7 +- borgmatic/actions/info.py | 7 +- borgmatic/actions/list.py | 7 +- borgmatic/actions/mount.py | 7 +- borgmatic/actions/restore.py | 10 ++- borgmatic/actions/rinfo.py | 7 +- borgmatic/actions/rlist.py | 7 +- borgmatic/borg/break_lock.py | 6 +- borgmatic/borg/create.py | 4 +- borgmatic/borg/export_tar.py | 6 +- borgmatic/borg/extract.py | 6 +- borgmatic/borg/info.py | 3 +- borgmatic/borg/rinfo.py | 3 +- borgmatic/borg/rlist.py | 3 +- borgmatic/borg/version.py | 3 +- borgmatic/commands/arguments.py | 14 ++-- borgmatic/commands/borgmatic.py | 48 ++++++++++--- borgmatic/config/load.py | 1 + borgmatic/config/normalize.py | 10 ++- borgmatic/config/override.py | 9 ++- borgmatic/execute.py | 6 +- borgmatic/hooks/mysql.py | 4 +- borgmatic/hooks/postgresql.py | 11 ++- setup.cfg | 11 +-- test_requirements.txt | 36 +++++----- tests/unit/actions/test_check.py | 6 +- tests/unit/actions/test_restore.py | 10 ++- tests/unit/borg/test_borg.py | 5 +- tests/unit/borg/test_break_lock.py | 29 ++++++-- tests/unit/borg/test_check.py | 62 ++++++++++++++--- tests/unit/borg/test_create.py | 20 ++++-- tests/unit/borg/test_extract.py | 14 +++- tests/unit/borg/test_flags.py | 27 ++++++-- tests/unit/borg/test_info.py | 9 ++- tests/unit/borg/test_list.py | 28 ++++++-- tests/unit/borg/test_mount.py | 11 ++- tests/unit/borg/test_rcreate.py | 98 +++++++++++++++++++++++---- tests/unit/borg/test_rinfo.py | 72 ++++++++++++++++---- tests/unit/borg/test_rlist.py | 29 ++++++-- tests/unit/borg/test_version.py | 3 +- tests/unit/commands/test_borgmatic.py | 3 +- tests/unit/config/test_normalize.py | 40 ++++++++--- tests/unit/config/test_validate.py | 7 +- tests/unit/hooks/test_cronhub.py | 6 +- tests/unit/hooks/test_cronitor.py | 6 +- tests/unit/hooks/test_healthchecks.py | 4 +- tests/unit/hooks/test_mongodb.py | 3 +- tests/unit/hooks/test_mysql.py | 17 ++++- tests/unit/test_execute.py | 6 +- 50 files changed, 603 insertions(+), 155 deletions(-) diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index fbf1d49d..3d2998b6 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_borg( - repository, storage, local_borg_version, borg_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + borg_arguments, + local_path, + remote_path, ): ''' Run the "borg" action for the given repository. diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index eb6e4547..2174161c 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -7,7 +7,12 @@ logger = logging.getLogger(__name__) def run_break_lock( - repository, storage, local_borg_version, break_lock_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + break_lock_arguments, + local_path, + remote_path, ): ''' Run the "break-lock" action for the given repository. diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index 6cd775f5..54023127 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -9,7 +9,12 @@ logger = logging.getLogger(__name__) def run_info( - repository, storage, local_borg_version, info_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + info_arguments, + local_path, + remote_path, ): ''' Run the "info" action for the given repository and archive. diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 0e736f84..359f3b67 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_list( - repository, storage, local_borg_version, list_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + list_arguments, + local_path, + remote_path, ): ''' Run the "list" action for the given repository and archive. diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index 6e787b18..72e321a0 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_mount( - repository, storage, local_borg_version, mount_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + mount_arguments, + local_path, + remote_path, ): ''' Run the "mount" action for the given repository. diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 1e8d175d..f061dca8 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -114,7 +114,13 @@ def restore_single_database( def collect_archive_database_names( - repository, archive, location, storage, local_borg_version, local_path, remote_path, + repository, + archive, + location, + storage, + local_borg_version, + local_path, + remote_path, ): ''' Given a local or remote repository path, a resolved archive name, a location configuration dict, @@ -180,7 +186,7 @@ def find_databases_to_restore(requested_database_names, archive_database_names): if 'all' in restore_names[UNSPECIFIED_HOOK]: restore_names[UNSPECIFIED_HOOK].remove('all') - for (hook_name, database_names) in archive_database_names.items(): + for hook_name, database_names in archive_database_names.items(): restore_names.setdefault(hook_name, []).extend(database_names) # If a database is to be restored as part of "all", then remove it from restore names so diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 001a0851..0947ec3d 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_rinfo( - repository, storage, local_borg_version, rinfo_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + rinfo_arguments, + local_path, + remote_path, ): ''' Run the "rinfo" action for the given repository. diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index c2ea1c63..10d06a51 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) def run_rlist( - repository, storage, local_borg_version, rlist_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ): ''' Run the "rlist" action for the given repository. diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 2dff31ec..7099af83 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -7,7 +7,11 @@ logger = logging.getLogger(__name__) def break_lock( - repository_path, storage_config, local_borg_version, local_path='borg', remote_path=None, + repository_path, + storage_config, + local_borg_version, + local_path='borg', + remote_path=None, ): ''' Given a local or remote repository path, a storage configuration dict, the local Borg version, diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 58c3509b..8782dc6b 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -509,7 +509,9 @@ def create_archive( ) elif output_log_level is None: return execute_command_and_capture_output( - create_command, working_directory=working_directory, extra_environment=borg_environment, + create_command, + working_directory=working_directory, + extra_environment=borg_environment, ) else: execute_command( diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 2b427f03..a624f07d 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -45,7 +45,11 @@ def export_tar_archive( + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + flags.make_repository_archive_flags(repository_path, archive, local_borg_version,) + + flags.make_repository_archive_flags( + repository_path, + archive, + local_borg_version, + ) + (destination_path,) + (tuple(paths) if paths else ()) ) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index a95685cf..f9471416 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -108,7 +108,11 @@ def extract_archive( + (('--strip-components', str(strip_components)) if strip_components else ()) + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) - + flags.make_repository_archive_flags(repository, archive, local_borg_version,) + + flags.make_repository_archive_flags( + repository, + archive, + local_borg_version, + ) + (tuple(paths) if paths else ()) ) diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index 80739c07..ef2c0c44 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -62,7 +62,8 @@ def display_archives_info( if info_arguments.json: return execute_command_and_capture_output( - full_command, extra_environment=environment.make_environment(storage_config), + full_command, + extra_environment=environment.make_environment(storage_config), ) else: execute_command( diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index 8a30b7f0..97d7a666 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -50,7 +50,8 @@ def display_repository_info( if rinfo_arguments.json: return execute_command_and_capture_output( - full_command, extra_environment=extra_environment, + full_command, + extra_environment=extra_environment, ) else: execute_command( diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 9686b67b..c051a9ad 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -40,7 +40,8 @@ def resolve_archive_name( ) output = execute_command_and_capture_output( - full_command, extra_environment=environment.make_environment(storage_config), + full_command, + extra_environment=environment.make_environment(storage_config), ) try: latest_archive = output.strip().splitlines()[-1] diff --git a/borgmatic/borg/version.py b/borgmatic/borg/version.py index 6d6c302c..d90a7aae 100644 --- a/borgmatic/borg/version.py +++ b/borgmatic/borg/version.py @@ -19,7 +19,8 @@ def local_borg_version(storage_config, local_path='borg'): + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) ) output = execute_command_and_capture_output( - full_command, extra_environment=environment.make_environment(storage_config), + full_command, + extra_environment=environment.make_environment(storage_config), ) try: diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index b89ca761..61b54769 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -178,7 +178,9 @@ def make_parsers(): help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)', ) global_group.add_argument( - '--log-file', type=str, help='Write log messages to this file instead of syslog', + '--log-file', + type=str, + help='Write log messages to this file instead of syslog', ) global_group.add_argument( '--log-file-format', @@ -258,10 +260,13 @@ def make_parsers(): help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key (Borg 2.x+ only)', ) rcreate_group.add_argument( - '--append-only', action='store_true', help='Create an append-only repository', + '--append-only', + action='store_true', + help='Create an append-only repository', ) rcreate_group.add_argument( - '--storage-quota', help='Create a repository with a fixed storage quota', + '--storage-quota', + help='Create a repository with a fixed storage quota', ) rcreate_group.add_argument( '--make-parent-dirs', @@ -649,7 +654,8 @@ def make_parsers(): ) rlist_group = rlist_parser.add_argument_group('rlist arguments') rlist_group.add_argument( - '--repository', help='Path of repository to list, defaults to the configured repositories', + '--repository', + help='Path of repository to list, defaults to the configured repositories', ) rlist_group.add_argument( '--short', default=False, action='store_true', help='Output only archive names' diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5d56accb..fece475a 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -103,7 +103,9 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: repo_queue = Queue() for repo in location['repositories']: - repo_queue.put((repo, 0),) + repo_queue.put( + (repo, 0), + ) while not repo_queue.empty(): repository, retry_num = repo_queue.get() @@ -128,7 +130,9 @@ def run_configuration(config_filename, config, arguments): ) except (OSError, CalledProcessError, ValueError) as error: if retry_num < retries: - repo_queue.put((repository, retry_num + 1),) + repo_queue.put( + (repository, retry_num + 1), + ) tuple( # Consume the generator so as to trigger logging. log_error_records( f'{repository["path"]}: Error running actions for repository', @@ -279,7 +283,7 @@ def run_actions( **hook_context, ) - for (action_name, action_arguments) in arguments.items(): + for action_name, action_arguments in arguments.items(): if action_name == 'rcreate': borgmatic.actions.rcreate.run_rcreate( repository, @@ -408,19 +412,39 @@ def run_actions( ) elif action_name == 'rlist': yield from borgmatic.actions.rlist.run_rlist( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'list': yield from borgmatic.actions.list.run_list( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'rinfo': yield from borgmatic.actions.rinfo.run_rinfo( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'info': yield from borgmatic.actions.info.run_info( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'break-lock': borgmatic.actions.break_lock.run_break_lock( @@ -433,7 +457,12 @@ def run_actions( ) elif action_name == 'borg': borgmatic.actions.borg.run_borg( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) command.execute_hook( @@ -626,7 +655,8 @@ def collect_configuration_run_summary_logs(configs, arguments): logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") try: borg_umount.unmount_archive( - mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs), + mount_point=arguments['umount'].mount_point, + local_path=get_local_path(configs), ) except (CalledProcessError, OSError) as error: yield from log_error_records('Error unmounting mount point', error) diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 068e661f..f6290de8 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -121,6 +121,7 @@ def load_configuration(filename): Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError if there are too many recursive includes. ''' + # Use an embedded derived class for the include constructor so as to capture the filename # value. (functools.partial doesn't work for this use case because yaml.Constructor has to be # an actual class.) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index bcf088ac..147e4e4e 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -81,7 +81,10 @@ def normalize(config_filename, config): repository_path.partition('file://')[-1] ) config['location']['repositories'].append( - dict(repository_dict, path=updated_repository_path,) + dict( + repository_dict, + path=updated_repository_path, + ) ) elif repository_path.startswith('ssh://'): config['location']['repositories'].append(repository_dict) @@ -97,7 +100,10 @@ def normalize(config_filename, config): ) ) config['location']['repositories'].append( - dict(repository_dict, path=rewritten_repository_path,) + dict( + repository_dict, + path=rewritten_repository_path, + ) ) else: config['location']['repositories'].append(repository_dict) diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 8b2a1ab8..aacf375d 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -57,7 +57,12 @@ def parse_overrides(raw_overrides): for raw_override in raw_overrides: try: raw_keys, value = raw_override.split('=', 1) - parsed_overrides.append((tuple(raw_keys.split('.')), convert_value_type(value),)) + parsed_overrides.append( + ( + tuple(raw_keys.split('.')), + convert_value_type(value), + ) + ) except ValueError: raise ValueError( f"Invalid override '{raw_override}'. Make sure you use the form: SECTION.OPTION=VALUE" @@ -75,5 +80,5 @@ def apply_overrides(config, raw_overrides): ''' overrides = parse_overrides(raw_overrides) - for (keys, value) in overrides: + for keys, value in overrides: set_values(config, keys, value) diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 0afa5cca..39691dac 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -236,7 +236,11 @@ def execute_command( def execute_command_and_capture_output( - full_command, capture_stderr=False, shell=False, extra_environment=None, working_directory=None, + full_command, + capture_stderr=False, + shell=False, + extra_environment=None, + working_directory=None, ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index aeeeee5b..793b78b0 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -100,7 +100,9 @@ def execute_dump_command( dump.create_named_pipe_for_dump(dump_filename) return execute_command( - dump_command, extra_environment=extra_environment, run_to_completion=False, + dump_command, + extra_environment=extra_environment, + run_to_completion=False, ) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index a982209c..bcc48efd 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -123,7 +123,12 @@ def dump_databases(databases, log_prefix, location_config, dry_run): continue command = ( - (dump_command, '--no-password', '--clean', '--if-exists',) + ( + dump_command, + '--no-password', + '--clean', + '--if-exists', + ) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) @@ -146,7 +151,9 @@ def dump_databases(databases, log_prefix, location_config, dry_run): if dump_format == 'directory': dump.create_parent_directory_for_dump(dump_filename) execute_command( - command, shell=True, extra_environment=extra_environment, + command, + shell=True, + extra_environment=extra_environment, ) else: dump.create_named_pipe_for_dump(dump_filename) diff --git a/setup.cfg b/setup.cfg index f97ae972..30841386 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,18 +8,21 @@ filterwarnings = ignore:Deprecated call to `pkg_resources.declare_namespace\('ruamel'\)`.*:DeprecationWarning [flake8] -ignore = E501,W503 +max-line-length = 100 +extend-ignore = E203,E501,W503 exclude = *.*/* multiline-quotes = ''' docstring-quotes = ''' [tool:isort] -force_single_line = False -include_trailing_comma = True +profile=black known_first_party = borgmatic line_length = 100 -multi_line_output = 3 skip = .tox [codespell] skip = .git,.tox,build + +[pycodestyle] +ignore = E203 +max_line_length = 100 diff --git a/test_requirements.txt b/test_requirements.txt index 7910788b..67e02563 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,27 +1,29 @@ appdirs==1.4.4; python_version >= '3.8' -attrs==20.3.0; python_version >= '3.8' -black==19.10b0; python_version >= '3.8' -click==7.1.2; python_version >= '3.8' +attrs==22.2.0; python_version >= '3.8' +black==23.3.0; python_version >= '3.8' +chardet==5.1.0 +click==8.1.3; python_version >= '3.8' codespell==2.2.4 -colorama==0.4.4 -coverage==5.3 -flake8==4.0.1 +colorama==0.4.6 +coverage==7.2.3 +flake8==6.0.0 flake8-quotes==3.3.2 flake8-use-fstring==1.4 flake8-variables-names==0.0.5 -flexmock==0.10.4 -isort==5.9.1 -mccabe==0.6.1 -pluggy==0.13.1 -pathspec==0.8.1; python_version >= '3.8' -py==1.10.0 -pycodestyle==2.8.0 -pyflakes==2.4.0 -jsonschema==3.2.0 -pytest==7.2.0 +flexmock==0.11.3 +idna==3.4 +isort==5.12.0 +mccabe==0.7.0 +pluggy==1.0.0 +pathspec==0.11.1; python_version >= '3.8' +py==1.11.0 +pycodestyle==2.10.0 +pyflakes==3.0.1 +jsonschema==4.17.3 +pytest==7.3.0 pytest-cov==4.0.0 regex; python_version >= '3.8' -requests==2.25.0 +requests==2.28.2 ruamel.yaml>0.15.0,<0.18.0 toml==0.10.2; python_version >= '3.8' typed-ast; python_version >= '3.8' diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 4c2027eb..05f63b6a 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -12,7 +12,11 @@ def test_run_check_calls_hooks_for_configured_repository(): flexmock(module.borgmatic.borg.check).should_receive('check_archives').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( - repository=None, progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock(), + repository=None, + progress=flexmock(), + repair=flexmock(), + only=flexmock(), + force=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 667a23c3..16fe2920 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -148,7 +148,8 @@ def test_find_databases_to_restore_without_requested_names_finds_all_archive_dat archive_database_names = {'postresql_databases': ['foo', 'bar']} restore_names = module.find_databases_to_restore( - requested_database_names=[], archive_database_names=archive_database_names, + requested_database_names=[], + archive_database_names=archive_database_names, ) assert restore_names == archive_database_names @@ -158,7 +159,8 @@ def test_find_databases_to_restore_with_all_in_requested_names_finds_all_archive archive_database_names = {'postresql_databases': ['foo', 'bar']} restore_names = module.find_databases_to_restore( - requested_database_names=['all'], archive_database_names=archive_database_names, + requested_database_names=['all'], + archive_database_names=archive_database_names, ) assert restore_names == archive_database_names @@ -194,7 +196,9 @@ def test_ensure_databases_found_with_all_databases_found_does_not_raise(): def test_ensure_databases_found_with_no_databases_raises(): with pytest.raises(ValueError): module.ensure_databases_found( - restore_names={'postgresql_databases': []}, remaining_restore_names={}, found_names=[], + restore_names={'postgresql_databases': []}, + remaining_restore_names={}, + found_names=[], ) diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 545da6c0..5b735960 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -222,7 +222,10 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): ) module.run_arbitrary_borg( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=[], + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + options=[], ) diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 210aceeb..509fc1b8 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -10,7 +10,9 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock(command): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - command, borg_local_path='borg', extra_environment=None, + command, + borg_local_path='borg', + extra_environment=None, ).once() @@ -19,7 +21,9 @@ def test_break_lock_calls_borg_with_required_flags(): insert_execute_command_mock(('borg', 'break-lock', 'repo')) module.break_lock( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', ) @@ -28,7 +32,10 @@ def test_break_lock_calls_borg_with_remote_path_flags(): insert_execute_command_mock(('borg', 'break-lock', '--remote-path', 'borg1', 'repo')) module.break_lock( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + remote_path='borg1', ) @@ -37,7 +44,9 @@ def test_break_lock_calls_borg_with_umask_flags(): insert_execute_command_mock(('borg', 'break-lock', '--umask', '0770', 'repo')) module.break_lock( - repository_path='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={'umask': '0770'}, + local_borg_version='1.2.3', ) @@ -46,7 +55,9 @@ def test_break_lock_calls_borg_with_lock_wait_flags(): insert_execute_command_mock(('borg', 'break-lock', '--lock-wait', '5', 'repo')) module.break_lock( - repository_path='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={'lock_wait': '5'}, + local_borg_version='1.2.3', ) @@ -56,7 +67,9 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.break_lock( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', ) @@ -66,5 +79,7 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags(): insert_logging_mock(logging.DEBUG) module.break_lock( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 65e17291..1f992d3d 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -79,7 +79,12 @@ def test_parse_frequency_parses_into_timedeltas(frequency, expected_result): @pytest.mark.parametrize( - 'frequency', ('sometime', 'x days', '3 decades',), + 'frequency', + ( + 'sometime', + 'x days', + '3 decades', + ), ) def test_parse_frequency_raises_on_parse_error(frequency): with pytest.raises(ValueError): @@ -211,7 +216,10 @@ def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flags = module.make_check_flags('1.2.3', {}, ('data',)) - assert flags == ('--archives-only', '--verify-data',) + assert flags == ( + '--archives-only', + '--verify-data', + ) def test_make_check_flags_with_extract_omits_extract_flag(): @@ -227,7 +235,14 @@ def test_make_check_flags_with_repository_and_data_checks_does_not_return_reposi flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('repository', 'data',)) + flags = module.make_check_flags( + '1.2.3', + {}, + ( + 'repository', + 'data', + ), + ) assert flags == ('--verify-data',) @@ -236,7 +251,12 @@ def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags() flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo',) + flags = module.make_check_flags( + '1.2.3', + {}, + ('repository', 'archives'), + prefix='foo', + ) assert flags == ('--match-archives', 'sh:foo*') @@ -246,7 +266,10 @@ def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags(): flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', ) assert flags == ('--match-archives', 'sh:foo*') @@ -257,7 +280,10 @@ def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_retur flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', ) assert flags == ('--glob-archives', 'foo*') @@ -447,7 +473,11 @@ def test_check_archives_calls_borg_with_parameters(checks): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', {}, checks, check_last, prefix=None, + '1.2.3', + {}, + checks, + check_last, + prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) @@ -601,7 +631,11 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', {}, checks, check_last, prefix=None, + '1.2.3', + {}, + checks, + check_last, + prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1', 'check', 'repo')) @@ -628,7 +662,11 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', {}, checks, check_last, prefix=None, + '1.2.3', + {}, + checks, + check_last, + prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo')) @@ -656,7 +694,11 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', storage_config, checks, check_last, None, + '1.2.3', + storage_config, + checks, + check_last, + None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo')) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 42fedd56..818ce276 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -1053,7 +1053,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), + 'feature_available,option_flag', + ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), ) def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_parameters( feature_available, option_flag @@ -1188,7 +1189,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner')), + 'feature_available,option_flag', + ((True, '--numeric-ids'), (False, '--numeric-owner')), ) def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( feature_available, option_flag @@ -1290,7 +1292,12 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter @pytest.mark.parametrize( 'option_name,option_value', - (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),), + ( + ('ctime', True), + ('ctime', False), + ('birthtime', True), + ('birthtime', False), + ), ) def test_create_archive_with_basic_option_calls_borg_with_corresponding_parameter( option_name, option_value @@ -1766,7 +1773,12 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--info', '--progress',), + ('borg', 'create') + + REPO_ARCHIVE_WITH_PATHS + + ( + '--info', + '--progress', + ), output_log_level=logging.INFO, output_file=module.DO_NOT_CAPTURE, borg_local_path='borg', diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index de29cf4d..26fd7380 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -11,7 +11,9 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock(command, working_directory=None): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - command, working_directory=working_directory, extra_environment=None, + command, + working_directory=working_directory, + extra_environment=None, ).once() @@ -152,7 +154,11 @@ def test_extract_archive_calls_borg_with_remote_path_parameters(): @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner'),), + 'feature_available,option_flag', + ( + (True, '--numeric-ids'), + (False, '--numeric-owner'), + ), ) def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available, option_flag): flexmock(module.os.path).should_receive('abspath').and_return('repo') @@ -441,7 +447,9 @@ def test_extract_archive_skips_abspath_for_remote_repository(): flexmock(module.os.path).should_receive('abspath').never() flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'extract', 'server:repo::archive'), working_directory=None, extra_environment=None, + ('borg', 'extract', 'server:repo::archive'), + working_directory=None, + extra_environment=None, ).once() flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 86247040..2eaff0ae 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -70,7 +70,11 @@ def test_make_repository_archive_flags_with_borg_features_separates_repository_a assert module.make_repository_archive_flags( repository_path='repo', archive='archive', local_borg_version='1.2.3' - ) == ('--repo', 'repo', 'archive',) + ) == ( + '--repo', + 'repo', + 'archive', + ) def test_make_repository_archive_flags_with_borg_features_joins_repository_and_archive(): @@ -86,9 +90,24 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a ( (None, None, True, ()), (None, '', True, ()), - ('re:foo-.*', '{hostname}-{now}', True, ('--match-archives', 're:foo-.*'),), # noqa: FS003 - ('sh:foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003 - ('foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003 + ( + 're:foo-.*', + '{hostname}-{now}', + True, + ('--match-archives', 're:foo-.*'), + ), # noqa: FS003 + ( + 'sh:foo-*', + '{hostname}-{now}', + False, + ('--glob-archives', 'foo-*'), + ), # noqa: FS003 + ( + 'foo-*', + '{hostname}-{now}', + False, + ('--glob-archives', 'foo-*'), + ), # noqa: FS003 ( None, '{hostname}-docs-{now}', # noqa: FS003 diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index d51bf14d..2eed4fea 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -69,7 +69,8 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'info', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.INFO) @@ -120,7 +121,8 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'info', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.DEBUG) @@ -145,7 +147,8 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'info', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') json_output = module.display_archives_info( diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 37af65a3..0a7db4cc 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -387,7 +387,8 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module.rlist).should_receive('make_rlist_command').and_return(('borg', 'list', 'repo')) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list', 'repo'), extra_environment=None, + ('borg', 'list', 'repo'), + extra_environment=None, ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').and_return( ('borg', 'list', 'repo::archive1') @@ -518,9 +519,18 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos @pytest.mark.parametrize( - 'archive_filter_flag', ('prefix', 'match_archives', 'sort_by', 'first', 'last',), + 'archive_filter_flag', + ( + 'prefix', + 'match_archives', + 'sort_by', + 'first', + 'last', + ), ) -def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_flag,): +def test_list_archive_with_archive_ignores_archive_filter_flag( + archive_filter_flag, +): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.logger).answer = lambda message: None @@ -566,7 +576,14 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(archive_filter_fl @pytest.mark.parametrize( - 'archive_filter_flag', ('prefix', 'match_archives', 'sort_by', 'first', 'last',), + 'archive_filter_flag', + ( + 'prefix', + 'match_archives', + 'sort_by', + 'first', + 'last', + ), ) def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes_it_to_rlist( archive_filter_flag, @@ -597,7 +614,8 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes ).and_return(('borg', 'rlist', '--repo', 'repo')) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rlist', '--repo', 'repo'), extra_environment=None, + ('borg', 'rlist', '--repo', 'repo'), + extra_environment=None, ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').with_args( diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 6161a249..658b2e52 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -10,7 +10,9 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock(command): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - command, borg_local_path='borg', extra_environment=None, + command, + borg_local_path='borg', + extra_environment=None, ).once() @@ -33,7 +35,12 @@ def test_mount_archive_calls_borg_with_required_flags(): def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_archives_flags(): flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) insert_execute_command_mock( ('borg', 'mount', '--repo', 'repo', '--match-archives', 'archive', '/mnt') ) diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index e232df22..4da04dff 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -36,7 +36,12 @@ def test_create_repository_calls_borg_with_flags(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -51,7 +56,12 @@ def test_create_repository_with_dry_run_skips_borg_call(): insert_rinfo_command_not_found_mock() flexmock(module).should_receive('execute_command').never() flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=True, @@ -65,7 +75,12 @@ def test_create_repository_with_dry_run_skips_borg_call(): def test_create_repository_raises_for_borg_rcreate_error(): insert_rinfo_command_not_found_mock() flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').and_raise( module.subprocess.CalledProcessError(2, 'borg rcreate') @@ -84,7 +99,12 @@ def test_create_repository_raises_for_borg_rcreate_error(): def test_create_repository_skips_creation_when_repository_already_exists(): insert_rinfo_command_found_mock() flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -114,7 +134,12 @@ def test_create_repository_with_source_repository_calls_borg_with_other_repo_fla insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--other-repo', 'other.borg', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -130,7 +155,12 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--copy-crypt-key', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -146,7 +176,12 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--append-only', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -162,7 +197,12 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--storage-quota', '5G', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -178,7 +218,12 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--make-parent-dirs', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -195,7 +240,12 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): insert_rcreate_command_mock(RCREATE_COMMAND + ('--info', '--repo', 'repo')) insert_logging_mock(logging.INFO) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -211,7 +261,12 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): insert_rcreate_command_mock(RCREATE_COMMAND + ('--debug', '--repo', 'repo')) insert_logging_mock(logging.DEBUG) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -226,7 +281,12 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(('borg1',) + RCREATE_COMMAND[1:] + ('--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -242,7 +302,12 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--remote-path', 'borg1', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, @@ -258,7 +323,12 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options insert_rinfo_command_not_found_mock() insert_rcreate_command_mock(RCREATE_COMMAND + ('--extra', '--options', '--repo', 'repo')) flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) module.create_repository( dry_run=False, diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index ec0819cf..979b253e 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -11,7 +11,12 @@ def test_display_repository_info_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--repo', 'repo'), @@ -53,7 +58,12 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--info', '--repo', 'repo'), @@ -74,10 +84,16 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'rinfo', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.INFO) @@ -95,7 +111,12 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter( flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--debug', '--show-rc', '--repo', 'repo'), @@ -117,10 +138,16 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'rinfo', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') insert_logging_mock(logging.DEBUG) @@ -138,10 +165,16 @@ def test_display_repository_info_with_json_calls_borg_with_json_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + ('borg', 'rinfo', '--json', '--repo', 'repo'), + extra_environment=None, ).and_return('[]') json_output = module.display_repository_info( @@ -158,7 +191,12 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg1', 'rinfo', '--repo', 'repo'), @@ -180,7 +218,12 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_pa flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--remote-path', 'borg1', '--repo', 'repo'), @@ -203,7 +246,12 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parame flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'rinfo', '--lock-wait', '5', '--repo', 'repo'), diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 6b1561a2..b83ba615 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -29,7 +29,8 @@ def test_resolve_archive_name_calls_borg_with_parameters(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') assert ( @@ -42,7 +43,8 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') insert_logging_mock(logging.INFO) @@ -56,7 +58,8 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter( expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') insert_logging_mock(logging.DEBUG) @@ -70,7 +73,8 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return(expected_archive + '\n') assert ( @@ -100,7 +104,8 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param def test_resolve_archive_name_without_archives_raises(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, ).and_return('') with pytest.raises(ValueError): @@ -374,7 +379,15 @@ def test_make_rlist_command_includes_short(): @pytest.mark.parametrize( 'argument_name', - ('sort_by', 'first', 'last', 'exclude', 'exclude_from', 'pattern', 'patterns_from',), + ( + 'sort_by', + 'first', + 'last', + 'exclude', + 'exclude_from', + 'pattern', + 'patterns_from', + ), ) def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) @@ -411,7 +424,9 @@ def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_p None, None, '1.2.3' ).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'foo-*', None, '1.2.3', + 'foo-*', + None, + '1.2.3', ).and_return(('--match-archives', 'foo-*')) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) diff --git a/tests/unit/borg/test_version.py b/tests/unit/borg/test_version.py index 66789a8c..a051f693 100644 --- a/tests/unit/borg/test_version.py +++ b/tests/unit/borg/test_version.py @@ -15,7 +15,8 @@ def insert_execute_command_and_capture_output_mock( ): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( - command, extra_environment=None, + command, + extra_environment=None, ).once().and_return(version_output) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 56f2332c..bd98c01f 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -229,7 +229,8 @@ def test_run_configuration_retries_hard_error(): ).and_return([flexmock()]) error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', OSError, + 'foo: Error running actions for repository', + OSError, ).and_return(error_logs) config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 1d61a771..63e3187a 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -21,13 +21,21 @@ from borgmatic.config import normalize as module {'location': {'source_directories': ['foo', 'bar']}}, False, ), - ({'location': None}, {'location': None}, False,), + ( + {'location': None}, + {'location': None}, + False, + ), ( {'storage': {'compression': 'yes_please'}}, {'storage': {'compression': 'yes_please'}}, False, ), - ({'storage': None}, {'storage': None}, False,), + ( + {'storage': None}, + {'storage': None}, + False, + ), ( {'hooks': {'healthchecks': 'https://example.com'}}, {'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}}, @@ -48,10 +56,9 @@ from borgmatic.config import normalize as module {'hooks': {'cronhub': {'ping_url': 'https://example.com'}}}, False, ), - ({'hooks': None}, {'hooks': None}, False,), ( - {'consistency': {'checks': ['archives']}}, - {'consistency': {'checks': [{'name': 'archives'}]}}, + {'hooks': None}, + {'hooks': None}, False, ), ( @@ -59,9 +66,26 @@ from borgmatic.config import normalize as module {'consistency': {'checks': [{'name': 'archives'}]}}, False, ), - ({'consistency': None}, {'consistency': None}, False,), - ({'location': {'numeric_owner': False}}, {'location': {'numeric_ids': False}}, False,), - ({'location': {'bsd_flags': False}}, {'location': {'flags': False}}, False,), + ( + {'consistency': {'checks': ['archives']}}, + {'consistency': {'checks': [{'name': 'archives'}]}}, + False, + ), + ( + {'consistency': None}, + {'consistency': None}, + False, + ), + ( + {'location': {'numeric_owner': False}}, + {'location': {'numeric_ids': False}}, + False, + ), + ( + {'location': {'bsd_flags': False}}, + {'location': {'flags': False}}, + False, + ), ( {'storage': {'remote_rate_limit': False}}, {'storage': {'upload_rate_limit': False}}, diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 11f3127c..24105547 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -138,7 +138,6 @@ def test_guard_configuration_contains_repository_does_not_raise_when_repository_ def test_guard_configuration_contains_repository_does_not_raise_when_repository_label_in_config(): - module.guard_configuration_contains_repository( repository='repo', configurations={ @@ -190,13 +189,15 @@ def test_guard_single_repository_selected_raises_when_multiple_repositories_conf def test_guard_single_repository_selected_does_not_raise_when_single_repository_configured_and_none_selected(): module.guard_single_repository_selected( - repository=None, configurations={'config.yaml': {'location': {'repositories': ['repo']}}}, + repository=None, + configurations={'config.yaml': {'location': {'repositories': ['repo']}}}, ) def test_guard_single_repository_selected_does_not_raise_when_no_repositories_configured_and_one_selected(): module.guard_single_repository_selected( - repository='repo', configurations={'config.yaml': {'location': {'repositories': []}}}, + repository='repo', + configurations={'config.yaml': {'location': {'repositories': []}}}, ) diff --git a/tests/unit/hooks/test_cronhub.py b/tests/unit/hooks/test_cronhub.py index f470b88e..2941592b 100644 --- a/tests/unit/hooks/test_cronhub.py +++ b/tests/unit/hooks/test_cronhub.py @@ -108,5 +108,9 @@ def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, ) diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py index 7ec1e2e6..12b96855 100644 --- a/tests/unit/hooks/test_cronitor.py +++ b/tests/unit/hooks/test_cronitor.py @@ -93,5 +93,9 @@ def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, ) diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index c975e4fd..5c6977da 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -206,7 +206,9 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): payload = 'data' flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload) flexmock(module.requests).should_receive('post').with_args( - f"https://hc-ping.com/{hook_config['ping_url']}", data=payload.encode('utf-8'), verify=True, + f"https://hc-ping.com/{hook_config['ping_url']}", + data=payload.encode('utf-8'), + verify=True, ).and_return(flexmock(ok=True)) module.ping_monitor( diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 77b830b7..f038a881 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -114,7 +114,8 @@ def test_dump_databases_runs_mongodump_with_directory_format(): flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').with_args( - ['mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'], shell=True, + ['mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'], + shell=True, ).and_return(flexmock()).once() assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [] diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 137a88a5..da5da16d 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -149,7 +149,14 @@ def test_execute_dump_command_runs_mysqldump(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--add-drop-database', '--databases', 'foo', '--result-file', 'dump',), + ( + 'mysqldump', + '--add-drop-database', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment=None, run_to_completion=False, ).and_return(process).once() @@ -175,7 +182,13 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( - ('mysqldump', '--databases', 'foo', '--result-file', 'dump',), + ( + 'mysqldump', + '--databases', + 'foo', + '--result-file', + 'dump', + ), extra_environment=None, run_to_completion=False, ).and_return(process).once() diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 14924078..a6dd9d86 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -320,7 +320,11 @@ def test_execute_command_and_capture_output_returns_output_with_extra_environmen expected_output = '[]' flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( - full_command, stderr=None, shell=False, env={'a': 'b', 'c': 'd'}, cwd=None, + full_command, + stderr=None, + shell=False, + env={'a': 'b', 'c': 'd'}, + cwd=None, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( From 8cb5a42a9e156f126faf3797fff55a8bf95f4b61 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 21:21:25 -0700 Subject: [PATCH 082/344] Drop deprecated pkg_resources in favor of importlib.metadata and packaging. --- borgmatic/borg/feature.py | 26 +++++++++++++------------- borgmatic/commands/borgmatic.py | 8 ++++++-- borgmatic/config/validate.py | 17 +++++++++++++++-- setup.cfg | 2 -- setup.py | 1 + test_requirements.txt | 4 ++++ tests/unit/config/test_validate.py | 22 ++++++++++++++++++++++ 7 files changed, 61 insertions(+), 19 deletions(-) diff --git a/borgmatic/borg/feature.py b/borgmatic/borg/feature.py index 5294121d..b9311cd1 100644 --- a/borgmatic/borg/feature.py +++ b/borgmatic/borg/feature.py @@ -1,6 +1,6 @@ from enum import Enum -from pkg_resources import parse_version +from packaging.version import parse class Feature(Enum): @@ -18,17 +18,17 @@ class Feature(Enum): FEATURE_TO_MINIMUM_BORG_VERSION = { - Feature.COMPACT: parse_version('1.2.0a2'), # borg compact - Feature.ATIME: parse_version('1.2.0a7'), # borg create --atime - Feature.NOFLAGS: parse_version('1.2.0a8'), # borg create --noflags - Feature.NUMERIC_IDS: parse_version('1.2.0b3'), # borg create/extract/mount --numeric-ids - Feature.UPLOAD_RATELIMIT: parse_version('1.2.0b3'), # borg create --upload-ratelimit - Feature.SEPARATE_REPOSITORY_ARCHIVE: parse_version('2.0.0a2'), # --repo with separate archive - Feature.RCREATE: parse_version('2.0.0a2'), # borg rcreate - Feature.RLIST: parse_version('2.0.0a2'), # borg rlist - Feature.RINFO: parse_version('2.0.0a2'), # borg rinfo - Feature.MATCH_ARCHIVES: parse_version('2.0.0b3'), # borg --match-archives - Feature.EXCLUDED_FILES_MINUS: parse_version('2.0.0b5'), # --list --filter uses "-" for excludes + Feature.COMPACT: parse('1.2.0a2'), # borg compact + Feature.ATIME: parse('1.2.0a7'), # borg create --atime + Feature.NOFLAGS: parse('1.2.0a8'), # borg create --noflags + Feature.NUMERIC_IDS: parse('1.2.0b3'), # borg create/extract/mount --numeric-ids + Feature.UPLOAD_RATELIMIT: parse('1.2.0b3'), # borg create --upload-ratelimit + Feature.SEPARATE_REPOSITORY_ARCHIVE: parse('2.0.0a2'), # --repo with separate archive + Feature.RCREATE: parse('2.0.0a2'), # borg rcreate + Feature.RLIST: parse('2.0.0a2'), # borg rlist + Feature.RINFO: parse('2.0.0a2'), # borg rinfo + Feature.MATCH_ARCHIVES: parse('2.0.0b3'), # borg --match-archives + Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'), # --list --filter uses "-" for excludes } @@ -37,4 +37,4 @@ def available(feature, borg_version): Given a Borg Feature constant and a Borg version string, return whether that feature is available in that version of Borg. ''' - return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse_version(borg_version) + return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse(borg_version) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fece475a..999e9d8e 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -8,7 +8,11 @@ from queue import Queue from subprocess import CalledProcessError import colorama -import pkg_resources + +try: + import importlib_metadata +except ModuleNotFoundError: # pragma: nocover + import importlib.metadata as importlib_metadata import borgmatic.actions.borg import borgmatic.actions.break_lock @@ -706,7 +710,7 @@ def main(): # pragma: no cover global_arguments = arguments['global'] if global_arguments.version: - print(pkg_resources.require('borgmatic')[0].version) + print(importlib_metadata.version('borgmatic')) sys.exit(0) if global_arguments.bash_completion: print(borgmatic.commands.completion.bash_completion()) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index fcf29d38..537f4bee 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -1,9 +1,13 @@ import os import jsonschema -import pkg_resources import ruamel.yaml +try: + import importlib_metadata +except ModuleNotFoundError: # pragma: nocover + import importlib.metadata as importlib_metadata + from borgmatic.config import environment, load, normalize, override @@ -11,8 +15,17 @@ def schema_filename(): ''' Path to the installed YAML configuration schema file, used to validate and parse the configuration. + + Raise FileNotFoundError when the schema path does not exist. ''' - return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml') + try: + return next( + str(path.locate()) + for path in importlib_metadata.files('borgmatic') + if path.match('config/schema.yaml') + ) + except StopIteration: + raise FileNotFoundError('Configuration file schema could not be found') def format_json_error_path_element(path_element): diff --git a/setup.cfg b/setup.cfg index 30841386..a5ba3a6a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,8 +4,6 @@ description_file=README.md [tool:pytest] testpaths = tests addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end -filterwarnings = - ignore:Deprecated call to `pkg_resources.declare_namespace\('ruamel'\)`.*:DeprecationWarning [flake8] max-line-length = 100 diff --git a/setup.py b/setup.py index f6fca3db..ce6b78e5 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ setup( install_requires=( 'colorama>=0.4.1,<0.5', 'jsonschema', + 'packaging', 'requests', 'ruamel.yaml>0.15.0,<0.18.0', 'setuptools', diff --git a/test_requirements.txt b/test_requirements.txt index 67e02563..6516a500 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -12,8 +12,10 @@ flake8-use-fstring==1.4 flake8-variables-names==0.0.5 flexmock==0.11.3 idna==3.4 +importlib_metadata==6.3.0; python_version < '3.8' isort==5.12.0 mccabe==0.7.0 +packaging==23.1 pluggy==1.0.0 pathspec==0.11.1; python_version >= '3.8' py==1.11.0 @@ -27,3 +29,5 @@ requests==2.28.2 ruamel.yaml>0.15.0,<0.18.0 toml==0.10.2; python_version >= '3.8' typed-ast; python_version >= '3.8' +typing-extensions==4.5.0; python_version < '3.8' +zipp==3.15.0; python_version < '3.8' diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 24105547..e2b9f98f 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -4,6 +4,28 @@ from flexmock import flexmock from borgmatic.config import validate as module +def test_schema_filename_finds_schema_path(): + schema_path = '/var/borgmatic/config/schema.yaml' + + flexmock(module.importlib_metadata).should_receive('files').and_return( + flexmock(match=lambda path: False, locate=lambda: None), + flexmock(match=lambda path: True, locate=lambda: schema_path), + flexmock(match=lambda path: False, locate=lambda: None), + ) + + assert module.schema_filename() == schema_path + + +def test_schema_filename_with_missing_schema_path_raises(): + flexmock(module.importlib_metadata).should_receive('files').and_return( + flexmock(match=lambda path: False, locate=lambda: None), + flexmock(match=lambda path: False, locate=lambda: None), + ) + + with pytest.raises(FileNotFoundError): + assert module.schema_filename() + + def test_format_json_error_path_element_formats_array_index(): module.format_json_error_path_element(3) == '[3]' From 7ee37a890ed66d8529b93f5fb27d79ac3ee5a0d8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 23:22:07 -0700 Subject: [PATCH 083/344] Fix broken end-to-end tests by no longer using an editable package there, a work-around for https://github.com/pypa/packaging-problems/issues/609 --- tests/end-to-end/docker-compose.yaml | 1 + tox.ini | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index e6b0bf2b..0bbec8cc 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -23,6 +23,7 @@ services: - "../..:/app:ro" tmpfs: - "/app/borgmatic.egg-info" + - "/app/build" tty: true working_dir: /app entrypoint: /app/scripts/run-full-tests diff --git a/tox.ini b/tox.ini index b81588af..3a2b4764 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,9 @@ commands = pytest {posargs} [testenv:end-to-end] +usedevelop = False deps = -rtest_requirements.txt + . passenv = COVERAGE_FILE commands = pytest {posargs} --no-cov tests/end-to-end From 28e62d824b0f1a23dbb4e861b9576f61e0f2aa8a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 23:28:07 -0700 Subject: [PATCH 084/344] Upgrade end-to-end test packages. --- scripts/run-full-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run-full-tests b/scripts/run-full-tests index bf26c212..fb611fc4 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -21,8 +21,8 @@ apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client m py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true -python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 -pip3 install --ignore-installed tox==3.25.1 +python3 -m pip install --no-cache --upgrade pip==23.0.1 setuptools==67.6.1 +pip3 install --ignore-installed tox==4.4.12 export COVERAGE_FILE=/tmp/.coverage if [ "$1" != "--end-to-end-only" ] ; then From 25506b8d2c7fde14427957ec33d93de10dd3d999 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Apr 2023 23:47:51 -0700 Subject: [PATCH 085/344] Backing out upgrade of end-to-end test packages, because apparently we can't have nice things. --- scripts/run-full-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run-full-tests b/scripts/run-full-tests index fb611fc4..bf26c212 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -21,8 +21,8 @@ apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client m py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true -python3 -m pip install --no-cache --upgrade pip==23.0.1 setuptools==67.6.1 -pip3 install --ignore-installed tox==4.4.12 +python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 +pip3 install --ignore-installed tox==3.25.1 export COVERAGE_FILE=/tmp/.coverage if [ "$1" != "--end-to-end-only" ] ; then From 991e08f16dc99693b32826c6cd21401c32b06633 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Apr 2023 09:13:13 -0700 Subject: [PATCH 086/344] Add Unraid borgmatic installation link to docs. --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 917eb437..ca703163 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -96,7 +96,7 @@ installing borgmatic: * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/) * [NixOS](https://search.nixos.org/packages?show=borgmatic&sort=relevance&type=packages&query=borgmatic) * [Ansible role](https://github.com/borgbase/ansible-role-borgbackup) - * [virtualenv](https://virtualenv.pypa.io/en/stable/) + * [Unraid](https://unraid.net/community/apps?q=borgmatic#r) ## Hosting providers From 9f5769f87be3e75a3b86ee1a6f31a6754ec2ed4e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 16 Apr 2023 15:41:17 -0700 Subject: [PATCH 087/344] Make docs/schema a little more container agnostic / less Docker specific. --- borgmatic/config/schema.yaml | 14 +++++++------- docs/how-to/backup-your-databases.md | 10 +++++----- docs/how-to/develop-on-borgmatic.md | 23 +++++++++++++---------- docs/how-to/set-up-backups.md | 4 ++-- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 26500855..c4fd3138 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -836,25 +836,25 @@ properties: Command to use instead of "pg_dump" or "pg_dumpall". This can be used to run a specific pg_dump version (e.g., one inside a running - docker container). Defaults to "pg_dump" for - single database dump or "pg_dumpall" to dump - all databases. + container). Defaults to "pg_dump" for single + database dump or "pg_dumpall" to dump all + databases. example: docker exec my_pg_container pg_dump pg_restore_command: type: string description: | Command to use instead of "pg_restore". This can be used to run a specific pg_restore - version (e.g., one inside a running docker - container). Defaults to "pg_restore". + version (e.g., one inside a running container). + Defaults to "pg_restore". example: docker exec my_pg_container pg_restore psql_command: type: string description: | Command to use instead of "psql". This can be used to run a specific psql version (e.g., - one inside a running docker container). - Defaults to "psql". + one inside a running container). Defaults to + "psql". example: docker exec my_pg_container psql options: type: string diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 91dba18f..db93c073 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -138,7 +138,7 @@ hooks: ### Containers -If your database is running within a Docker container and borgmatic is too, no +If your database is running within a container and borgmatic is too, no problem—simply configure borgmatic to connect to the container's name on its exposed port. For instance: @@ -154,10 +154,10 @@ hooks: But what if borgmatic is running on the host? You can still connect to a database container if its ports are properly exposed to the host. For -instance, when running the database container with Docker, you can specify -`--publish 127.0.0.1:5433:5432` so that it exposes the container's port 5432 -to port 5433 on the host (only reachable on localhost, in this case). Or the -same thing with Docker Compose: +instance, when running the database container, you can specify `--publish +127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433 +on the host (only reachable on localhost, in this case). Or the same thing +with Docker Compose: ```yaml services: diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 027a8145..19b4bd18 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -87,19 +87,20 @@ tox -e codespell borgmatic additionally includes some end-to-end tests that integration test with Borg and supported databases for a few representative scenarios. These tests don't run by default when running `tox`, because they're relatively slow -and depend on Docker containers for runtime dependencies. These tests tests do -run on the continuous integration (CI) server, and running them on your -developer machine is the closest thing to CI test parity. +and depend on containers for runtime dependencies. These tests do run on the +continuous integration (CI) server, and running them on your developer machine +is the closest thing to CI-test parity. -If you would like to run the full test suite, first install Docker and [Docker -Compose](https://docs.docker.com/compose/install/). Then run: +If you would like to run the full test suite, first install Docker (or Podman; +see below) and [Docker Compose](https://docs.docker.com/compose/install/). +Then run: ```bash scripts/run-end-to-end-dev-tests ``` -Note that this scripts assumes you have permission to run Docker. If you -don't, then you may need to run with `sudo`. +This script assumes you have permission to run `docker`. If you don't, then +you may need to run with `sudo`. #### Podman @@ -119,6 +120,7 @@ some key points to double-check: * Create a non-root Podman socket for that user: ```bash systemctl --user enable --now podman.socket + systemctl --user start --now podman.socket ``` Then you'll be able to run end-to-end tests as per normal, and the test script @@ -161,11 +163,12 @@ To build and view a copy of the documentation with your local changes, run the following from the root of borgmatic's source code: ```bash -sudo scripts/dev-docs +scripts/dev-docs ``` -This requires Docker to be installed on your system. You may not need to use -sudo if your non-root user has permissions to run Docker. +This requires Docker (or Podman; see below) to be installed on your system. +This script assumes you have permission to run `docker`. If you don't, then +you may need to run with `sudo`. After you run the script, you can point your web browser at http://localhost:8080 to view the documentation with your changes. diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index ca703163..93402828 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -82,8 +82,8 @@ on a relatively dedicated system, then a global install can work out fine. Besides the approaches described above, there are several other options for installing borgmatic: - * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files) - * [Docker image with multi-arch and Docker CLI support](https://hub.docker.com/r/modem7/borgmatic-docker/) + * [container image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files) + * [container image with multi-arch and Docker CLI support](https://hub.docker.com/r/modem7/borgmatic-docker/) * [Debian](https://tracker.debian.org/pkg/borgmatic) * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic) * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic) From 8bb7631f50998d9b0cd796f3c098162011bc1bc7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Apr 2023 21:22:51 -0700 Subject: [PATCH 088/344] Fix missing mock in unit test. --- tests/unit/borg/test_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 818ce276..d2ab397d 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -2550,7 +2550,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) - flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module).should_receive('check_all_source_directories_exist').and_raise(ValueError) with pytest.raises(ValueError): module.create_archive( From 3b21ce4ce8b8af640ccc690f2ec0b193a1adc9cb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Apr 2023 21:43:08 -0700 Subject: [PATCH 089/344] Rename "master" development branch to "main" to use more inclusive language (#684). --- .drone.yml | 2 +- NEWS | 4 +++- README.md | 2 +- docs/how-to/develop-on-borgmatic.md | 2 +- docs/how-to/set-up-backups.md | 6 +++--- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.drone.yml b/.drone.yml index 9a7630ac..406d7e09 100644 --- a/.drone.yml +++ b/.drone.yml @@ -53,6 +53,6 @@ trigger: repo: - borgmatic-collective/borgmatic branch: - - master + - main event: - push diff --git a/NEWS b/NEWS index 4a27d074..740cc6f8 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,8 @@ * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas + * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to + update your development checkouts accordingly. 1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. @@ -367,7 +369,7 @@ * #398: Clarify canonical home of borgmatic in documentation. * #406: Clarify that spaces in path names should not be backslashed in path names. * #423: Fix error handling to error loudly when Borg gets killed due to running out of memory! - * Fix build so as not to attempt to build and push documentation for a non-master branch. + * Fix build so as not to attempt to build and push documentation for a non-main branch. * "Fix" build failure with Alpine Edge by switching from Edge to Alpine 3.13. * Move #borgmatic IRC channel from Freenode to Libera Chat due to Freenode takeover drama. IRC connection info: https://torsion.org/borgmatic/#issues diff --git a/README.md b/README.md index eb827ae9..2af792ba 100644 --- a/README.md +++ b/README.md @@ -165,5 +165,5 @@ Also, please check out the [borgmatic development how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for info on cloning source code, running tests, etc. -![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/master) +![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/main) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 19b4bd18..6d2b13b5 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -7,7 +7,7 @@ eleventyNavigation: --- ## Source code -To get set up to hack on borgmatic, first clone master via HTTPS or SSH: +To get set up to hack on borgmatic, first clone it via HTTPS or SSH: ```bash git clone https://projects.torsion.org/borgmatic-collective/borgmatic.git diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 93402828..be229cb2 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -279,7 +279,7 @@ that, you can configure a separate job runner to invoke it periodically. ### cron If you're using cron, download the [sample cron -file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/master/sample/cron/borgmatic). +file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/main/sample/cron/borgmatic). Then, from the directory where you downloaded it: ```bash @@ -303,9 +303,9 @@ you may already have borgmatic systemd service and timer files. If so, you may be able to skip some of the steps below.) First, download the [sample systemd service -file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.service) +file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/main/sample/systemd/borgmatic.service) and the [sample systemd timer -file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer). +file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/main/sample/systemd/borgmatic.timer). Then, from the directory where you downloaded them: From 269fac074b1f9b6558f25dc3a33dc446e6bb653a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Apr 2023 23:14:51 -0700 Subject: [PATCH 090/344] Attempt to use Podman-in-Podman for building docs instead of Docker-in-Podman. --- .drone.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.drone.yml b/.drone.yml index 406d7e09..5a8bb56c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -38,16 +38,17 @@ clone: steps: - name: build - image: plugins/docker - settings: - username: + image: quay.io/stable/podman + environment: + USERNAME: from_secret: docker_username - password: + PASSWORD: from_secret: docker_password - registry: projects.torsion.org - repo: projects.torsion.org/borgmatic-collective/borgmatic - tags: docs - dockerfile: docs/Dockerfile + IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs + commands: + - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman push "$IMAGE_NAME" trigger: repo: From 0a6f5452f41e289c452e32cdd1aee93706e98d68 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Apr 2023 23:16:15 -0700 Subject: [PATCH 091/344] Fix broken Podman image name. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5a8bb56c..5137bacd 100644 --- a/.drone.yml +++ b/.drone.yml @@ -38,7 +38,7 @@ clone: steps: - name: build - image: quay.io/stable/podman + image: quay.io/podman/stable environment: USERNAME: from_secret: docker_username From 5f8c79dd164d144695f6385a0f4ed778f9e5daec Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 10:50:44 -0700 Subject: [PATCH 092/344] Attempt to get Podman-in-Podman builds working. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5137bacd..05f344fe 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt "unmask=/proc/*" --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 53ee0fcfada8d61ceefe652f109d63bf9f174b50 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:06:15 -0700 Subject: [PATCH 093/344] Another attempt at Podman-in-Podman incantations. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 05f344fe..1041e464 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt "unmask=/proc/*" --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt label=disable --security-opt unmask=ALL --device /dev/fuse --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From bb5028e4841c29834b63a19747c80bbb7df0a19c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:11:08 -0700 Subject: [PATCH 094/344] Sigh. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 1041e464..334962cb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt label=disable --security-opt unmask=ALL --device /dev/fuse --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt label=disable --device /dev/fuse --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From b85538c54c85bd21c326cb23255ad2f24397305f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:11:49 -0700 Subject: [PATCH 095/344] Double sigh. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 334962cb..5f4dbc94 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt label=disable --device /dev/fuse --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt label=disable --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 51bc53e5cade48f42c56ed8f707259e7c7b6d50e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:24:59 -0700 Subject: [PATCH 096/344] Whee. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5f4dbc94..3aa91923 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt label=disable --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 9e9a7c50e52a028adf9a20f1dc6ad123dcfc100f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:30:30 -0700 Subject: [PATCH 097/344] =?UTF-8?q?=F0=9F=98=8A=F0=9F=94=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 3aa91923..c4526039 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns-uid-map-group podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 4e78cf1b951a7abb6ec55fc341fe27f58f723e74 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:33:15 -0700 Subject: [PATCH 098/344] =?UTF-8?q?=E0=B2=A0=5F=E0=B2=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index c4526039..d95f0f00 100644 --- a/.drone.yml +++ b/.drone.yml @@ -47,7 +47,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns-uid-map-group podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 02eeca1fc2845efc331212856dcd150a1bd9dbe7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:36:30 -0700 Subject: [PATCH 099/344] Hmm. --- .drone.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index d95f0f00..0e03945e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,8 +46,10 @@ steps: from_secret: docker_password IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: + - cat /etc/subuid + - cat /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From f82bf619ff6ba15fc54aee05f7bf69c00c8d7964 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:41:35 -0700 Subject: [PATCH 100/344] More. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 0e03945e..59fa1752 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,7 +49,7 @@ steps: - cat /etc/subuid - cat /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From d6e1cef356e29285b4d129f2b55bfc0daba75576 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:49:43 -0700 Subject: [PATCH 101/344] Throwing stuff at the wall. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 59fa1752..53de5f87 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,7 +49,7 @@ steps: - cat /etc/subuid - cat /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns host --userns-uid-map-user podman --userns-gid-map-group podman --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 8a31c270780f3c3cbe036c950c8b175fa4ded957 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:50:25 -0700 Subject: [PATCH 102/344] To see what sticks. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 53de5f87..c3e30adf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,7 +49,7 @@ steps: - cat /etc/subuid - cat /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns host --userns-uid-map-user podman --userns-gid-map-group podman --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns host --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 1721c05d2e2b20c275f002bcd00955afa20a33d7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:52:23 -0700 Subject: [PATCH 103/344] Yet more. --- .drone.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index c3e30adf..48216fcd 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,10 +46,10 @@ steps: from_secret: docker_password IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - - cat /etc/subuid - - cat /etc/subgid + - echo "podman:100000:65536" > /etc/subuid + - echo "podman:100000:65536" > /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns host --volume /etc/subuid:/etc/subuid --volume /etc/subgid:/etc/subgid --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 4302a07c9b37e8d9ca3ca37e01e32f35b9c70c22 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:53:52 -0700 Subject: [PATCH 104/344] WTF. --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 48216fcd..cb26aef0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,8 +46,8 @@ steps: from_secret: docker_password IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - - echo "podman:100000:65536" > /etc/subuid - - echo "podman:100000:65536" > /etc/subgid + - echo "podman:100000:100000" > /etc/subuid + - echo "podman:100000:100000" > /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" From 499e42df359aa5c7442a0a1d4b2d6e41e89d7d7a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 11:58:06 -0700 Subject: [PATCH 105/344] =?UTF-8?q?=F0=9F=98=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index cb26aef0..7808e29f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,7 +49,7 @@ steps: - echo "podman:100000:100000" > /etc/subuid - echo "podman:100000:100000" > /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --security-opt label=disable --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From 7f7b89d79c733b1daa94b67ec4be3e17fb3514ec Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 12:03:51 -0700 Subject: [PATCH 106/344] Trying a different approach: Ditching Podman-in-Podman. --- .drone.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index 7808e29f..98b893e8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -32,13 +32,17 @@ steps: --- kind: pipeline name: documentation +type: exec + +platform: + os: linux + arch: amd64 clone: skip_verify: true steps: - name: build - image: quay.io/podman/stable environment: USERNAME: from_secret: docker_username @@ -46,10 +50,8 @@ steps: from_secret: docker_password IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - - echo "podman:100000:100000" > /etc/subuid - - echo "podman:100000:100000" > /etc/subgid - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --security-opt label=disable --isolation chroot --userns-uid-map-user podman --userns-gid-map-group podman --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile - podman push "$IMAGE_NAME" trigger: From f947525fcafbd34676c0eef91f2fa207ea7b9e0d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:45:26 -0700 Subject: [PATCH 107/344] ? --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 98b893e8..25f57985 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume .:/app - podman push "$IMAGE_NAME" trigger: From c9bf52ee45628669f3b0c44ad741091ee0ef8382 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:46:49 -0700 Subject: [PATCH 108/344] Sigh again. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 25f57985..53a61c56 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume .:/app + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app - podman push "$IMAGE_NAME" trigger: From 1e03046d9a72155123679d9786e318a198ff0f35 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:50:26 -0700 Subject: [PATCH 109/344] *Seriously?* --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 53a61c56..441859bf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" - podman push "$IMAGE_NAME" trigger: From 08edecacae473a2cf979d58f88531ead0ad54472 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:55:37 -0700 Subject: [PATCH 110/344] WTF?! --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 441859bf..ddbb5d73 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" --storage-opt ignore_chown_errors - podman push "$IMAGE_NAME" trigger: From 7ff994a96467b06894a44cb2ac8a2d376d9e4a67 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 13:56:12 -0700 Subject: [PATCH 111/344] =?UTF-8?q?=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index ddbb5d73..2e0e0b17 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" --storage-opt ignore_chown_errors + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" --storage-opt ignore_chown_errors=true - podman push "$IMAGE_NAME" trigger: From f2f6fb537a9cfbd96aaf5ad416a74fbcfdaa8c84 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 14:19:34 -0700 Subject: [PATCH 112/344] !!! --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 2e0e0b17..11788468 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,7 +51,7 @@ steps: IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs commands: - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org - - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --volume $(pwd):/app --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" --storage-opt ignore_chown_errors=true + - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" . - podman push "$IMAGE_NAME" trigger: From 065be1d9d4feb4236712394f63a0c5d18ab624cb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 14:28:04 -0700 Subject: [PATCH 113/344] More inclusive language. --- SECURITY.md | 4 ++-- borgmatic/borg/rlist.py | 4 ++-- borgmatic/config/generate.py | 2 +- borgmatic/config/schema.yaml | 4 ++-- docs/how-to/backup-your-databases.md | 12 ++++++------ docs/how-to/inspect-your-backups.md | 2 +- docs/how-to/make-per-application-backups.md | 8 ++++---- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index d82b6f32..64c3d3b3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,8 +7,8 @@ permalink: security-policy/index.html While we want to hear about security vulnerabilities in all versions of borgmatic, security fixes are only made to the most recently released version. -It's simply not practical for our small volunteer effort to maintain multiple -release branches and put out separate security patches for each. +It's not practical for our small volunteer effort to maintain multiple release +branches and put out separate security patches for each. ## Reporting a vulnerability diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index c051a9ad..7f468705 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -17,8 +17,8 @@ def resolve_archive_name( ): ''' Given a local or remote repository path, an archive name, a storage config dict, a local Borg - path, and a remote Borg path, simply return the archive name. But if the archive name is - "latest", then instead introspect the repository for the latest archive and return its name. + path, and a remote Borg path, return the archive name. But if the archive name is "latest", + then instead introspect the repository for the latest archive and return its name. Raise ValueError if "latest" is given but there are no archives in the repository. ''' diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index d486f23c..081186e3 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -260,7 +260,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi ) continue - # This is some sort of scalar. Simply set it into the destination. + # This is some sort of scalar. Set it into the destination. destination_config[field_name] = source_config[field_name] return destination_config diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index c4fd3138..0cf02b25 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -60,7 +60,7 @@ properties: or port. If systemd service is used, then add local repository paths in the systemd service file to the ReadWritePaths list. Prior to borgmatic 1.7.10, repositories - was just a list of plain path strings. + was a list of plain path strings. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver @@ -1216,7 +1216,7 @@ properties: type: string description: | Healthchecks ping URL or UUID to notify when a - backup begins, ends, errors or just to send logs. + backup begins, ends, errors, or to send only logs. example: https://hc-ping.com/your-uuid-here verify_tls: type: boolean diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index db93c073..0a770d61 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -139,8 +139,8 @@ hooks: ### Containers If your database is running within a container and borgmatic is too, no -problem—simply configure borgmatic to connect to the container's name on its -exposed port. For instance: +problem—configure borgmatic to connect to the container's name on its exposed +port. For instance: ```yaml hooks: @@ -179,7 +179,7 @@ hooks: password: trustsome1 ``` -Of course, alter the ports in these examples to suit your particular database +You can alter the ports in these examples to suit your particular database system. @@ -397,9 +397,9 @@ dumps with any database system. With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors when borgmatic tries to connect to your database, a natural reaction is to increase your borgmatic verbosity with `--verbosity 2` and go looking in the -logs. You'll notice however that your database password does not show up in -the logs. This is likely not the cause of the authentication problem unless -you mistyped your password, however; borgmatic passes your password to the +logs. You'll notice though that your database password does not show up in the +logs. This is likely not the cause of the authentication problem unless you +mistyped your password, however; borgmatic passes your password to the database via an environment variable that does not appear in the logs. The cause of an authentication error is often on the database side—in the diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 73020eda..e89d23f6 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -169,7 +169,7 @@ brackets. For instance, the default log format is: `[{asctime}] {levelname}: {message}`. This means each log message is recorded as the log time (in square brackets), a logging level name, a colon, and the actual log message. -So if you just want each log message to get logged *without* a timestamp or a +So if you only want each log message to get logged *without* a timestamp or a logging level name: ```bash diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 9ee93f44..f2ddf012 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -86,8 +86,8 @@ uses the `archive_name_format` option to automatically limit which archives get used for actions operating on multiple archives. This prevents, for instance, duplicate archives from showing up in `rlist` or `info` results—even if the same repository appears in multiple borgmatic configuration files. To -take advantage of this feature, simply use a different `archive_name_format` -in each configuration file. +take advantage of this feature, use a different `archive_name_format` in each +configuration file. Under the hood, borgmatic accomplishes this by substituting globs for certain ephemeral data placeholders in your `archive_name_format`—and using the result @@ -108,8 +108,8 @@ archives used for actions like `rlist`, `info`, `prune`, `check`, etc. The end result is that when borgmatic runs the actions for a particular application-specific configuration file, it only operates on the archives -created for that application. Of course, this doesn't apply to actions like -`compact` that operate on an entire repository. +created for that application. But this doesn't apply to actions like `compact` +that operate on an entire repository. If this behavior isn't quite smart enough for your needs, you can use the `match_archives` option to override the pattern that borgmatic uses for From ee5c25f3bd9402b47738be1c7cfed5f282930ed7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 21:44:42 -0700 Subject: [PATCH 114/344] Add additional tests for PostgreSQL hook fixes (#678). --- NEWS | 5 ++ tests/unit/hooks/test_postgresql.py | 122 ++++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index 740cc6f8..69938b0f 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,11 @@ * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas + * #678: Fix error from PostgreSQL when dumping a database with a "format" of "plain". + * #678: Fix PostgreSQL hook to support "psql_command" and "pg_restore_command" options containing + commands with arguments. + * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break + database dumping. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 62b10d16..e3ecbdb6 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -76,7 +76,16 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam def test_database_names_to_dump_with_all_and_format_lists_databases_with_username(): database = {'name': 'all', 'format': 'custom', 'username': 'postgres'} flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--username', 'postgres'), + ( + 'psql', + '--list', + '--no-password', + '--no-psqlrc', + '--csv', + '--tuples-only', + '--username', + 'postgres', + ), extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') @@ -110,6 +119,28 @@ def test_database_names_to_dump_with_all_and_format_excludes_particular_database ) +def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): + database = {'name': 'all', 'format': 'custom', 'psql_command': 'docker exec mycontainer psql'} + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'docker', + 'exec', + 'mycontainer', + 'psql', + '--list', + '--no-password', + '--no-psqlrc', + '--csv', + '--tuples-only', + ), + extra_environment=object, + ).and_return('foo,text').once() + + assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + 'foo', + ) + + def test_dump_databases_runs_pg_dump_for_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] @@ -434,7 +465,16 @@ def test_restore_database_dump_runs_pg_restore(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -632,12 +672,45 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): ) +def test_restore_database_dump_runs_psql_for_plain_database_dump(): + database_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on', '--dbname', 'foo'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + + module.restore_database_dump( + database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + ) + + def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): database_config = [ { 'name': 'foo', - 'pg_restore_command': 'special_pg_restore', - 'psql_command': 'special_psql', + 'pg_restore_command': 'docker exec mycontainer pg_restore', + 'psql_command': 'docker exec mycontainer psql', 'schemas': None, } ] @@ -648,7 +721,10 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( ( - 'special_pg_restore', + 'docker', + 'exec', + 'mycontainer', + 'pg_restore', '--no-password', '--if-exists', '--exit-on-error', @@ -662,7 +738,19 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('special_psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ( + 'docker', + 'exec', + 'mycontainer', + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -707,7 +795,16 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--no-psqlrc', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), extra_environment={'PGSSLMODE': 'disable'}, ).once() @@ -743,7 +840,16 @@ def test_restore_database_dump_with_schemas_restores_schemas(): extra_environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( - ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'), + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), extra_environment={'PGSSLMODE': 'disable'}, ).once() From a14870ce4899a8aa38a2f38832bffe74b8490b78 Mon Sep 17 00:00:00 2001 From: Jesse Johnson Date: Wed, 19 Apr 2023 18:47:22 -0700 Subject: [PATCH 115/344] Expand source directories when checking for existence (#682). --- borgmatic/borg/create.py | 2 +- tests/unit/borg/test_create.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 8782dc6b..7413dfc5 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -314,7 +314,7 @@ def check_all_source_directories_exist(source_directories): missing_directories = [ source_directory for source_directory in source_directories - if not os.path.exists(source_directory) + if not all([os.path.exists(directory) for directory in expand_directory(source_directory)]) ] if missing_directories: raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}") diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index d2ab397d..5011f48b 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -2565,3 +2565,18 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ storage_config={}, local_borg_version='1.2.3', ) + + +def test_check_all_source_directories_exist_with_glob_and_tilde_directories(): + flexmock(module).should_receive('expand_directory').with_args('foo*').and_return( + ('foo', 'food') + ) + flexmock(module).should_receive('expand_directory').with_args('~/bar').and_return( + ('/root/bar',) + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.os.path).should_receive('exists').with_args('foo').and_return(True) + flexmock(module.os.path).should_receive('exists').with_args('food').and_return(True) + flexmock(module.os.path).should_receive('exists').with_args('/root/bar').and_return(True) + + module.check_all_source_directories_exist(['foo*', '~/bar']) From b555fcb95621ba46d1bc6d2208cbac79bd3e020c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 23:08:21 -0700 Subject: [PATCH 116/344] Add "source_directories_must_exist" expansion fix to NEWS (#682). --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 69938b0f..20214674 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,7 @@ commands with arguments. * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break database dumping. + * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. From 9ca31530a0d06196f2e3a64fb7c47ff05f89bf7d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 23:15:22 -0700 Subject: [PATCH 117/344] Add missing test for check_all_source_directories_exist() raising. --- tests/unit/borg/test_create.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 5011f48b..3728a0bf 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -2580,3 +2580,11 @@ def test_check_all_source_directories_exist_with_glob_and_tilde_directories(): flexmock(module.os.path).should_receive('exists').with_args('/root/bar').and_return(True) module.check_all_source_directories_exist(['foo*', '~/bar']) + + +def test_check_all_source_directories_exist_with_non_existent_directory_raises(): + flexmock(module).should_receive('expand_directory').with_args('foo').and_return(('foo',)) + flexmock(module.os.path).should_receive('exists').and_return(False) + + with pytest.raises(ValueError): + module.check_all_source_directories_exist(['foo']) From 71b75800cdf770fa6af519ae493fe5be8413b0a8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Apr 2023 23:32:57 -0700 Subject: [PATCH 118/344] Get more verbose in the end-to-end test restore. --- tests/end-to-end/test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 30aea4a8..e0c929f3 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -118,7 +118,7 @@ def test_database_dump_and_restore(): # Restore the database from the archive. subprocess.check_call( - ['borgmatic', '--config', config_path, 'restore', '--archive', archive_name] + ['borgmatic', '--config', '-v', '2', config_path, 'restore', '--archive', archive_name] ) finally: os.chdir(original_working_directory) From 3cefeaa229959c8e6bea0adf299f1fd513b6c6ee Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Apr 2023 09:30:08 -0700 Subject: [PATCH 119/344] Fix end-to-end test command-line syntax. --- tests/end-to-end/test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index e0c929f3..81e7c5e7 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -118,7 +118,7 @@ def test_database_dump_and_restore(): # Restore the database from the archive. subprocess.check_call( - ['borgmatic', '--config', '-v', '2', config_path, 'restore', '--archive', archive_name] + ['borgmatic', '-v, '2', '--config', config_path, 'restore', '--archive', archive_name] ) finally: os.chdir(original_working_directory) From ae12ccd8e6b0f0a009b2214f93b1181fb4d00842 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Apr 2023 09:31:37 -0700 Subject: [PATCH 120/344] And fixing again... --- tests/end-to-end/test_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 81e7c5e7..5c4e22cc 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -118,7 +118,7 @@ def test_database_dump_and_restore(): # Restore the database from the archive. subprocess.check_call( - ['borgmatic', '-v, '2', '--config', config_path, 'restore', '--archive', archive_name] + ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name] ) finally: os.chdir(original_working_directory) From 7e64f415ba74a6a491171181196b2d5a80c876da Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Apr 2023 10:03:29 -0700 Subject: [PATCH 121/344] Attempt to fix failing end-to-end database test that only fails in CI. --- .drone.yml | 9 +++++---- NEWS | 1 + tests/end-to-end/docker-compose.yaml | 1 + tests/end-to-end/test_database.py | 6 +++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.drone.yml b/.drone.yml index 11788468..f2c5f7d7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3,17 +3,18 @@ name: python-3-8-alpine-3-13 services: - name: postgresql - image: postgres:13.1-alpine + image: docker.io/postgres:13.1-alpine environment: + POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: test - name: mysql - image: mariadb:10.5 + image: docker.io/mariadb:10.5 environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test - name: mongodb - image: mongo:5.0.5 + image: docker.io/mongo:5.0.5 environment: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: test @@ -23,7 +24,7 @@ clone: steps: - name: build - image: alpine:3.13 + image: docker.io/alpine:3.13 environment: TEST_CONTAINER: true pull: always diff --git a/NEWS b/NEWS index 20214674..530a447f 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,7 @@ commands with arguments. * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break database dumping. + * #678: Fix calls to psql in PostgreSQL hook to abort on error during a database restore. * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 0bbec8cc..f9895a8c 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -3,6 +3,7 @@ services: postgresql: image: docker.io/postgres:13.1-alpine environment: + POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: test mysql: diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 5c4e22cc..8b38e071 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -36,17 +36,17 @@ hooks: postgresql_databases: - name: test hostname: postgresql - username: postgres + username: test password: test format: {postgresql_dump_format} - name: all hostname: postgresql - username: postgres + username: test password: test - name: all format: custom hostname: postgresql - username: postgres + username: test password: test mysql_databases: - name: test From 5962fd473e54ada92de6a220db729da11c769bfe Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Apr 2023 10:34:50 -0700 Subject: [PATCH 122/344] Another try. Backing out psql error changes (#678). --- .drone.yml | 2 +- NEWS | 1 - borgmatic/hooks/postgresql.py | 6 +----- tests/end-to-end/docker-compose.yaml | 1 - tests/end-to-end/test_database.py | 6 +++--- tests/unit/hooks/test_postgresql.py | 8 ++++++-- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.drone.yml b/.drone.yml index f2c5f7d7..dcc19c67 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,3 +1,4 @@ +--- kind: pipeline name: python-3-8-alpine-3-13 @@ -5,7 +6,6 @@ services: - name: postgresql image: docker.io/postgres:13.1-alpine environment: - POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: test - name: mysql diff --git a/NEWS b/NEWS index 530a447f..20214674 100644 --- a/NEWS +++ b/NEWS @@ -7,7 +7,6 @@ commands with arguments. * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break database dumping. - * #678: Fix calls to psql in PostgreSQL hook to abort on error during a database restore. * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 859f7ece..3325391f 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -229,11 +229,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, restore_command = ( tuple(psql_command if use_psql_command else pg_restore_command) + ('--no-password',) - + ( - ('--no-psqlrc', '--set', 'ON_ERROR_STOP=on') - if use_psql_command - else ('--if-exists', '--exit-on-error', '--clean') - ) + + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean')) + (('--dbname', database['name']) if not all_databases else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index f9895a8c..0bbec8cc 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -3,7 +3,6 @@ services: postgresql: image: docker.io/postgres:13.1-alpine environment: - POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: test mysql: diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 8b38e071..5c4e22cc 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -36,17 +36,17 @@ hooks: postgresql_databases: - name: test hostname: postgresql - username: test + username: postgres password: test format: {postgresql_dump_format} - name: all hostname: postgresql - username: test + username: postgres password: test - name: all format: custom hostname: postgresql - username: test + username: postgres password: test mysql_databases: - name: test diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index e3ecbdb6..b3a55fa4 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -656,7 +656,11 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( - ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on'), + ( + 'psql', + '--no-password', + '--no-psqlrc', + ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -680,7 +684,7 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( - ('psql', '--no-password', '--no-psqlrc', '--set', 'ON_ERROR_STOP=on', '--dbname', 'foo'), + ('psql', '--no-password', '--no-psqlrc', '--dbname', 'foo'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, From 22b84a2feaad01fe2a80e46e01c177e61bc80f8f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Apr 2023 10:07:40 -0700 Subject: [PATCH 123/344] Switch to Docker Compose for dev-docs script, so podman-docker is no longer needed for Podman users. --- docs/docker-compose.yaml | 22 ++++++++++++++++++++++ docs/how-to/develop-on-borgmatic.md | 5 ++--- scripts/dev-docs | 12 +++++++----- 3 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 docs/docker-compose.yaml diff --git a/docs/docker-compose.yaml b/docs/docker-compose.yaml new file mode 100644 index 00000000..e854c13a --- /dev/null +++ b/docs/docker-compose.yaml @@ -0,0 +1,22 @@ +version: '3' +services: + docs: + image: borgmatic-docs + container_name: docs + ports: + - 8080:80 + build: + dockerfile: docs/Dockerfile + context: .. + args: + ENVIRONMENT: dev + message: + image: alpine + container_name: message + command: + - sh + - -c + - | + echo "You can view dev docs at http://localhost:8080" + depends_on: + - docs diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 6d2b13b5..391d7950 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -113,8 +113,7 @@ borgmatic's end-to-end tests optionally support using Setting up Podman is outside the scope of this documentation, but here are some key points to double-check: - * Install Podman along with `podman-docker` and your desired networking - support. + * Install Podman and your desired networking support. * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the non-root user who will run tests. * Create a non-root Podman socket for that user: @@ -186,5 +185,5 @@ borgmatic's developer build for documentation optionally supports using [Podman](https://podman.io/) instead of Docker. Setting up Podman is outside the scope of this documentation. But once you -install `podman-docker`, then `scripts/dev-docs` should automatically use +install and configure Podman, then `scripts/dev-docs` should automatically use Podman instead of Docker. diff --git a/scripts/dev-docs b/scripts/dev-docs index 1ecc7e46..866ae0e9 100755 --- a/scripts/dev-docs +++ b/scripts/dev-docs @@ -2,8 +2,10 @@ set -e -docker build --tag borgmatic-docs --build-arg ENVIRONMENT=dev --file docs/Dockerfile . -echo -echo "You can view dev docs at http://localhost:8080" -echo -docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs +USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock + +if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then + export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH" +fi + +docker-compose --file docs/docker-compose.yaml up --build --force-recreate From 96aca4f446c427eb8cce4212e5e3f7df1e5f9744 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 24 Apr 2023 20:24:41 +0530 Subject: [PATCH 124/344] Updated existing tests to use new parameters Signed-off-by: Chirag Aggarwal --- borgmatic/borg/mount.py | 2 +- borgmatic/borg/prune.py | 4 ++- tests/unit/borg/test_mount.py | 66 ++++++++++++----------------------- tests/unit/borg/test_prune.py | 41 ++++++++++++++++++++-- 4 files changed, 65 insertions(+), 48 deletions(-) diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 2f6132da..764f72e2 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -34,7 +34,7 @@ def mount_archive( + flags.make_flags_from_arguments( mount_arguments, - excludes=('repository', 'archive', 'mount_point', 'path', 'options'), + excludes=('repository', 'archive', 'mount_point', 'paths', 'options'), ) + (('-o', mount_arguments.options) if mount_arguments.options else ()) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index c9a46354..ba41fba3 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -78,10 +78,12 @@ def prune_archives( + flags.make_flags_from_arguments( prune_arguments, - excludes=('repository', 'stats'), + excludes=('repository', 'stats', 'list_archives'), ) + + (('--list',) if prune_arguments.list_archives else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 658b2e52..1438ebe4 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -21,13 +21,11 @@ def test_mount_archive_calls_borg_with_required_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'mount', 'repo', '/mnt')) + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=False) module.mount_archive( repository_path='repo', archive=None, - mount_point='/mnt', - paths=None, - foreground=False, - options=None, + mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', ) @@ -45,13 +43,11 @@ def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_a ('borg', 'mount', '--repo', 'repo', '--match-archives', 'archive', '/mnt') ) + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=False) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=None, - foreground=False, - options=None, + mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', ) @@ -64,13 +60,11 @@ def test_mount_archive_without_archive_calls_borg_with_repository_flags_only(): ) insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt')) + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=False) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=None, - foreground=False, - options=None, + mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', ) @@ -83,13 +77,11 @@ def test_mount_archive_calls_borg_with_path_flags(): ) insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt', 'path1', 'path2')) + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=['path1', 'path2'], foreground=False) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=['path1', 'path2'], - foreground=False, - options=None, + mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', ) @@ -104,13 +96,11 @@ def test_mount_archive_calls_borg_with_remote_path_flags(): ('borg', 'mount', '--remote-path', 'borg1', 'repo::archive', '/mnt') ) + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=False) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=None, - foreground=False, - options=None, + mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', remote_path='borg1', @@ -124,13 +114,11 @@ def test_mount_archive_calls_borg_with_umask_flags(): ) insert_execute_command_mock(('borg', 'mount', '--umask', '0770', 'repo::archive', '/mnt')) + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=False) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=None, - foreground=False, - options=None, + mount_arguments=mount_arguments, storage_config={'umask': '0770'}, local_borg_version='1.2.3', ) @@ -143,13 +131,11 @@ def test_mount_archive_calls_borg_with_lock_wait_flags(): ) insert_execute_command_mock(('borg', 'mount', '--lock-wait', '5', 'repo::archive', '/mnt')) + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=False) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=None, - foreground=False, - options=None, + mount_arguments=mount_arguments, storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', ) @@ -163,13 +149,11 @@ def test_mount_archive_with_log_info_calls_borg_with_info_parameter(): insert_execute_command_mock(('borg', 'mount', '--info', 'repo::archive', '/mnt')) insert_logging_mock(logging.INFO) + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=False) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=None, - foreground=False, - options=None, + mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', ) @@ -183,13 +167,11 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_flags(): insert_execute_command_mock(('borg', 'mount', '--debug', '--show-rc', 'repo::archive', '/mnt')) insert_logging_mock(logging.DEBUG) + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=False) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=None, - foreground=False, - options=None, + mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', ) @@ -208,13 +190,11 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): extra_environment=None, ).once() + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=True) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=None, - foreground=True, - options=None, + mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', ) @@ -227,13 +207,11 @@ def test_mount_archive_calls_borg_with_options_flags(): ) insert_execute_command_mock(('borg', 'mount', '-o', 'super_mount', 'repo::archive', '/mnt')) + mount_arguments = flexmock(mount_point='/mnt', options='super_mount', paths=None, foreground=False) module.mount_archive( repository_path='repo', archive='archive', - mount_point='/mnt', - paths=None, - foreground=False, - options='super_mount', + mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 128bdc0a..2ed9c900 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -117,12 +117,14 @@ def test_prune_archives_calls_borg_with_parameters(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.INFO) + prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( dry_run=False, repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + prune_arguments=prune_arguments, ) @@ -134,12 +136,14 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): insert_execute_command_mock(PRUNE_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) + prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( repository_path='repo', storage_config={}, dry_run=False, retention_config=flexmock(), local_borg_version='1.2.3', + prune_arguments=prune_arguments, ) @@ -151,15 +155,34 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): insert_execute_command_mock(PRUNE_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) + prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( repository_path='repo', storage_config={}, dry_run=False, retention_config=flexmock(), local_borg_version='1.2.3', + prune_arguments=prune_arguments, ) +# def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): +# flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') +# flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER +# flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) +# flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) +# insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) + +# prune_arguments = flexmock(stats=False, list_archives=False) +# module.prune_archives( +# repository_path='repo', +# storage_config={}, +# dry_run=True, +# retention_config=flexmock(), +# local_borg_version='1.2.3', +# prune_arguments=prune_arguments, +# ) + def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -167,12 +190,14 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) + prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( repository_path='repo', storage_config={}, dry_run=True, retention_config=flexmock(), local_borg_version='1.2.3', + prune_arguments=prune_arguments, ) @@ -183,6 +208,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1',) + PRUNE_COMMAND[1:] + ('repo',), logging.INFO) + prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -190,6 +216,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): retention_config=flexmock(), local_borg_version='1.2.3', local_path='borg1', + prune_arguments=prune_arguments, ) @@ -200,6 +227,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters( flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) + prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -207,6 +235,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters( retention_config=flexmock(), local_borg_version='1.2.3', remote_path='borg1', + prune_arguments=prune_arguments, ) @@ -217,13 +246,14 @@ def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_ou flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) + prune_arguments = flexmock(stats=True, list_archives=False) module.prune_archives( dry_run=False, repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', - stats=True, + prune_arguments=prune_arguments, ) @@ -234,13 +264,14 @@ def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_out flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) + prune_arguments = flexmock(stats=False, list_archives=True) module.prune_archives( dry_run=False, repository_path='repo', storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', - list_archives=True, + prune_arguments=prune_arguments, ) @@ -252,12 +283,14 @@ def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077', 'repo'), logging.INFO) + prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( dry_run=False, repository_path='repo', storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', + prune_arguments=prune_arguments, ) @@ -269,12 +302,14 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) + prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( dry_run=False, repository_path='repo', storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', + prune_arguments=prune_arguments, ) @@ -285,10 +320,12 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO) + prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( dry_run=False, repository_path='repo', storage_config={'extra_borg_options': {'prune': '--extra --options'}}, retention_config=flexmock(), local_borg_version='1.2.3', + prune_arguments=prune_arguments, ) From 8aaba9bb0ade4f765c42206e076c36dcb56f245e Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 24 Apr 2023 20:43:34 +0530 Subject: [PATCH 125/344] Added new flags to prune test for review Signed-off-by: Chirag Aggarwal --- tests/unit/borg/test_prune.py | 41 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 2ed9c900..66cb7cd4 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -166,23 +166,6 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): ) -# def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): -# flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') -# flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER -# flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) -# flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) -# insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) - -# prune_arguments = flexmock(stats=False, list_archives=False) -# module.prune_archives( -# repository_path='repo', -# storage_config={}, -# dry_run=True, -# retention_config=flexmock(), -# local_borg_version='1.2.3', -# prune_arguments=prune_arguments, -# ) - def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -329,3 +312,27 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): local_borg_version='1.2.3', prune_arguments=prune_arguments, ) + + +def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flags(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock( + PRUNE_COMMAND + + ('--newer', '1d', '--newest', '1y', '--older', '1m', '--oldest', '1w', 'repo'), + logging.INFO, + ) + + prune_arguments = flexmock( + stats=False, list_archives=False, newer='1d', newest='1y', older='1m', oldest='1w' + ) + module.prune_archives( + dry_run=False, + repository_path='repo', + storage_config={}, + retention_config=flexmock(), + local_borg_version='1.2.3', + prune_arguments=prune_arguments, + ) From 32395e47f91b503f61aef592f26b2c4370b52938 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Mon, 24 Apr 2023 20:49:41 +0530 Subject: [PATCH 126/344] Added duplicate flags test for prune Signed-off-by: Chirag Aggarwal --- tests/integration/borg/test_commands.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 1afb0e0f..6049a263 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -106,3 +106,17 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): borgmatic.borg.info.display_archives_info( 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) ) + +def test_prune_archives_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('prune')['prune'] + flexmock(borgmatic.borg.prune).should_receive('execute_command').replace_with( + assert_command_does_not_duplicate_flags + ) + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.prune.prune_archives( + False, 'repo', {}, {}, '2.3.4', fuzz_argument(arguments, argument_name) + ) From 68ee9687f5e01bd5abbedb43e88f8fb02d1bcd5d Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 27 Apr 2023 22:27:23 +0530 Subject: [PATCH 127/344] Added tests for all subcommands and used black formatter Signed-off-by: Chirag Aggarwal --- borgmatic/borg/mount.py | 2 - borgmatic/borg/prune.py | 3 - borgmatic/commands/arguments.py | 80 ++++++++++++++++++------- tests/integration/borg/test_commands.py | 1 + tests/unit/borg/test_info.py | 49 +++++++++++++++ tests/unit/borg/test_mount.py | 55 ++++++++++++++++- tests/unit/borg/test_rlist.py | 42 +++++++++++++ tests/unit/borg/test_transfer.py | 47 +++++++++++++++ 8 files changed, 252 insertions(+), 27 deletions(-) diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 764f72e2..33026170 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -31,12 +31,10 @@ def mount_archive( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) - + flags.make_flags_from_arguments( mount_arguments, excludes=('repository', 'archive', 'mount_point', 'paths', 'options'), ) - + (('-o', mount_arguments.options) if mount_arguments.options else ()) + ( ( diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index ba41fba3..c7d13931 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -75,16 +75,13 @@ def prune_archives( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--stats',) if prune_arguments.stats and not dry_run else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) - + flags.make_flags_from_arguments( prune_arguments, excludes=('repository', 'stats', 'list_archives'), ) + (('--list',) if prune_arguments.list_archives else ()) - + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) - + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 77119585..4b19135e 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -327,16 +327,24 @@ def make_parsers(): '--last', metavar='N', help='Only transfer last N archives after other filters are applied' ) transfer_group.add_argument( - '--oldest', metavar='TIMESPAN', help='Transfer archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + '--oldest', + metavar='TIMESPAN', + help='Transfer archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]', ) transfer_group.add_argument( - '--newest', metavar='TIMESPAN', help='Transfer archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', + metavar='TIMESPAN', + help='Transfer archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]', ) transfer_group.add_argument( - '--older', metavar='TIMESPAN', help='Transfer archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--older', + metavar='TIMESPAN', + help='Transfer archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) transfer_group.add_argument( - '--newer', metavar='TIMESPAN', help='Transfer archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--newer', + metavar='TIMESPAN', + help='Transfer archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) transfer_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' @@ -365,16 +373,24 @@ def make_parsers(): '--list', dest='list_archives', action='store_true', help='List archives kept/pruned' ) prune_group.add_argument( - '--oldest', metavar='TIMESPAN', help='Prune archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + '--oldest', + metavar='TIMESPAN', + help='Prune archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]', ) prune_group.add_argument( - '--newest', metavar='TIMESPAN', help='Prune archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', + metavar='TIMESPAN', + help='Prune archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]', ) prune_group.add_argument( - '--older', metavar='TIMESPAN', help='Prune archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--older', + metavar='TIMESPAN', + help='Prune archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) prune_group.add_argument( - '--newer', metavar='TIMESPAN', help='Prune archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--newer', + metavar='TIMESPAN', + help='Prune archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') @@ -625,16 +641,24 @@ def make_parsers(): '--last', metavar='N', help='Mount last N archives after other filters are applied' ) mount_group.add_argument( - '--oldest', metavar='TIMESPAN', help='Mount archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + '--oldest', + metavar='TIMESPAN', + help='Mount archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]', ) mount_group.add_argument( - '--newest', metavar='TIMESPAN', help='Mount archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', + metavar='TIMESPAN', + help='Mount archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]', ) mount_group.add_argument( - '--older', metavar='TIMESPAN', help='Mount archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--older', + metavar='TIMESPAN', + help='Mount archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) mount_group.add_argument( - '--newer', metavar='TIMESPAN', help='Mount archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--newer', + metavar='TIMESPAN', + help='Mount archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) mount_group.add_argument('--options', dest='options', help='Extra Borg mount options') mount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') @@ -728,16 +752,24 @@ def make_parsers(): '--last', metavar='N', help='List last N archives after other filters are applied' ) rlist_group.add_argument( - '--oldest', metavar='TIMESPAN', help='List archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + '--oldest', + metavar='TIMESPAN', + help='List archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]', ) rlist_group.add_argument( - '--newest', metavar='TIMESPAN', help='List archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', + metavar='TIMESPAN', + help='List archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]', ) rlist_group.add_argument( - '--older', metavar='TIMESPAN', help='List archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--older', + metavar='TIMESPAN', + help='List archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) rlist_group.add_argument( - '--newer', metavar='TIMESPAN', help='List archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--newer', + metavar='TIMESPAN', + help='List archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) rlist_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') @@ -871,16 +903,24 @@ def make_parsers(): '--last', metavar='N', help='Show info for last N archives after other filters are applied' ) info_group.add_argument( - '--oldest', metavar='TIMESPAN', help='Show info for archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]' + '--oldest', + metavar='TIMESPAN', + help='Show info for archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]', ) info_group.add_argument( - '--newest', metavar='TIMESPAN', help='Show info for archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]' + '--newest', + metavar='TIMESPAN', + help='Show info for archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]', ) info_group.add_argument( - '--older', metavar='TIMESPAN', help='Show info for archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--older', + metavar='TIMESPAN', + help='Show info for archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) info_group.add_argument( - '--newer', metavar='TIMESPAN', help='Show info for archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]' + '--newer', + metavar='TIMESPAN', + help='Show info for archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]', ) info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 6049a263..6c330d6a 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -107,6 +107,7 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) ) + def test_prune_archives_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments('prune')['prune'] flexmock(borgmatic.borg.prune).should_receive('execute_command').replace_with( diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index 2eed4fea..d7bf7316 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -433,3 +433,52 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): archive=None, json=False, prefix=None, match_archives=None, **{argument_name: 'value'} ), ) + + +def test_display_archives_info_with_date_based_matching_calls_borg_with_date_based_flags(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( + ('--newer', '1d', '--newest', '1y', '--older', '1m', '--oldest', '1w') + ) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ( + 'borg', + 'info', + '--newer', + '1d', + '--newest', + '1y', + '--older', + '1m', + '--oldest', + '1w', + '--repo', + 'repo', + ), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + info_arguments = flexmock( + archive=None, + json=False, + prefix=None, + match_archives=None, + newer='1d', + newest='1y', + older='1m', + oldest='1w', + ) + module.display_archives_info( + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + info_arguments=info_arguments, + ) diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 1438ebe4..05ff72e0 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -77,7 +77,9 @@ def test_mount_archive_calls_borg_with_path_flags(): ) insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt', 'path1', 'path2')) - mount_arguments = flexmock(mount_point='/mnt', options=None, paths=['path1', 'path2'], foreground=False) + mount_arguments = flexmock( + mount_point='/mnt', options=None, paths=['path1', 'path2'], foreground=False + ) module.mount_archive( repository_path='repo', archive='archive', @@ -207,7 +209,9 @@ def test_mount_archive_calls_borg_with_options_flags(): ) insert_execute_command_mock(('borg', 'mount', '-o', 'super_mount', 'repo::archive', '/mnt')) - mount_arguments = flexmock(mount_point='/mnt', options='super_mount', paths=None, foreground=False) + mount_arguments = flexmock( + mount_point='/mnt', options='super_mount', paths=None, foreground=False + ) module.mount_archive( repository_path='repo', archive='archive', @@ -215,3 +219,50 @@ def test_mount_archive_calls_borg_with_options_flags(): storage_config={}, local_borg_version='1.2.3', ) + + +def test_mount_archive_with_date_based_matching_calls_borg_with_date_based_flags(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock( + ( + 'borg', + 'mount', + '--newer', + '1d', + '--newest', + '1y', + '--older', + '1m', + '--oldest', + '1w', + '--repo', + 'repo', + '--match-archives', + None, + '/mnt', + ) + ) + + mount_arguments = flexmock( + mount_point='/mnt', + options=None, + paths=None, + foreground=False, + newer='1d', + newest='1y', + older='1m', + oldest='1w', + ) + module.mount_archive( + repository_path='repo', + archive=None, + mount_arguments=mount_arguments, + storage_config={}, + local_borg_version='1.2.3', + ) diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index b83ba615..b6fadd18 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -506,3 +506,45 @@ def test_list_repository_with_json_returns_borg_output(): ) == json_output ) + + +def test_make_rlist_command_with_date_based_matching_calls_borg_with_date_based_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( + ('--newer', '1d', '--newest', '1y', '--older', '1m', '--oldest', '1w') + ) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock( + archive=None, + paths=None, + json=False, + prefix=None, + match_archives=None, + newer='1d', + newest='1y', + older='1m', + oldest='1w', + ), + ) + + assert command == ( + 'borg', + 'list', + '--newer', + '1d', + '--newest', + '1y', + '--older', + '1m', + '--oldest', + '1w', + 'repo', + ) diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 8f41bf5a..fd6cb197 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -386,3 +386,50 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla archive=None, progress=None, match_archives=None, source_repository='other' ), ) + + +def test_transfer_archives_with_date_based_matching_calls_borg_with_date_based_flags(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( + ('--newer', '1d', '--newest', '1y', '--older', '1m', '--oldest', '1w') + ) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ( + 'borg', + 'transfer', + '--newer', + '1d', + '--newest', + '1y', + '--older', + '1m', + '--oldest', + '1w', + '--repo', + 'repo', + ), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, + borg_local_path='borg', + extra_environment=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, + progress=None, + source_repository='other', + newer='1d', + newest='1y', + older='1m', + oldest='1w', + ), + ) From 0009471f67f5a00f03bcaa925d0d92988f7c85c4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 18:46:13 -0700 Subject: [PATCH 128/344] start work on completion --- borgmatic/commands/arguments.py | 6 ++++++ borgmatic/commands/borgmatic.py | 3 +++ borgmatic/commands/completion.py | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 61b54769..c7198856 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -207,6 +207,12 @@ def make_parsers(): action='store_true', help='Show bash completion script and exit', ) + global_group.add_argument( + '--fish-completion', + default=False, + action='store_true', + help='Show fish completion script and exit', + ) global_group.add_argument( '--version', dest='version', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 999e9d8e..d5fedba7 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -715,6 +715,9 @@ def main(): # pragma: no cover if global_arguments.bash_completion: print(borgmatic.commands.completion.bash_completion()) sys.exit(0) + if global_arguments.fish_completion: + print(borgmatic.commands.completion.fish_completion()) + sys.exit(0) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, parse_logs = load_configurations( diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 1fc976bc..4f262ddd 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -55,3 +55,25 @@ def bash_completion(): '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic', ) ) + +def fish_completion(): + ''' + Return a fish completion script for the borgmatic command. Produce this by introspecting + borgmatic's command-line argument parsers. + ''' + top_level_parser, subparsers = arguments.make_parsers() + global_flags = parser_flags(top_level_parser) + actions = ' '.join(subparsers.choices.keys()) + + # Avert your eyes. + return '\n'.join( + ( + 'function __borgmatic_check_version', + ' set this_script (cat "$BASH_SOURCE" 2> /dev/null)', + ' set installed_script (borgmatic --bash-completion 2> /dev/null)', + ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', + f' echo "{UPGRADE_MESSAGE}"', + ' end', + 'end', + 'function __borgmatic_complete', + )) From 28b152aedd5e460cb16009b339781789650e40e9 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 19:31:42 -0700 Subject: [PATCH 129/344] make upgrade message a template --- borgmatic/commands/completion.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 4f262ddd..13210b02 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,12 +1,13 @@ from borgmatic.commands import arguments -UPGRADE_MESSAGE = ''' -Your bash completions script is from a different version of borgmatic than is +def upgrade_message(language: str, upgrade_command: str, completion_file: str): + return f''' +Your {language} completions script is from a different version of borgmatic than is currently installed. Please upgrade your script so your completions match the command-line flags in your installed borgmatic! Try this to upgrade: - sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE" - source $BASH_SOURCE + {upgrade_command} + source {completion_file} ''' @@ -34,7 +35,7 @@ def bash_completion(): ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - f' then cat << EOF\n{UPGRADE_MESSAGE}\nEOF', + ' then cat << EOF\n{}\nEOF'.format(upgrade_message('bash', 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', '$BASH_SOURCE')), ' fi', '}', 'complete_borgmatic() {', @@ -69,11 +70,11 @@ def fish_completion(): return '\n'.join( ( 'function __borgmatic_check_version', - ' set this_script (cat "$BASH_SOURCE" 2> /dev/null)', - ' set installed_script (borgmatic --bash-completion 2> /dev/null)', + ' set this_script (status current-filename)', + ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - f' echo "{UPGRADE_MESSAGE}"', + ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee (status current-filename)', '(status current-filename)')), ' end', 'end', - 'function __borgmatic_complete', + # 'function __borgmatic_complete', )) From 5678f3a96e9731e3ff747de13440f53aa1572f5c Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 19:44:11 -0700 Subject: [PATCH 130/344] basic working version --- borgmatic/commands/completion.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 13210b02..de0991c9 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,4 +1,5 @@ from borgmatic.commands import arguments +import shlex def upgrade_message(language: str, upgrade_command: str, completion_file: str): return f''' @@ -76,5 +77,12 @@ def fish_completion(): ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee (status current-filename)', '(status current-filename)')), ' end', 'end', - # 'function __borgmatic_complete', - )) + ) + tuple( + '''complete -c borgmatic -n '__borgmatic_check_version' -a '%s' -d %s -f''' + % (action, shlex.quote(subparser.description)) + for action, subparser in subparsers.choices.items() + ) + ( + 'complete -c borgmatic -a "%s" -d "borgmatic actions" -f' % actions, + 'complete -c borgmatic -a "%s" -d "borgmatic global flags" -f' % global_flags, + ) + ) From 25b3db72a0d9bb47df8f44ad86c5a705eb8104ee Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 19:58:22 -0700 Subject: [PATCH 131/344] make more precise, fix the version check fn --- borgmatic/commands/completion.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index de0991c9..da62cccc 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -64,25 +64,29 @@ def fish_completion(): borgmatic's command-line argument parsers. ''' top_level_parser, subparsers = arguments.make_parsers() - global_flags = parser_flags(top_level_parser) actions = ' '.join(subparsers.choices.keys()) # Avert your eyes. return '\n'.join( ( 'function __borgmatic_check_version', - ' set this_script (status current-filename)', + ' set this_script (cat (status current-filename) 2> /dev/null)', ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee (status current-filename)', '(status current-filename)')), ' end', 'end', + '__borgmatic_check_version &', ) + tuple( - '''complete -c borgmatic -n '__borgmatic_check_version' -a '%s' -d %s -f''' + '''complete -c borgmatic -a '%s' -d %s -f''' % (action, shlex.quote(subparser.description)) for action, subparser in subparsers.choices.items() ) + ( 'complete -c borgmatic -a "%s" -d "borgmatic actions" -f' % actions, - 'complete -c borgmatic -a "%s" -d "borgmatic global flags" -f' % global_flags, + ) + tuple( + '''complete -c borgmatic -a '%s' -d %s -f''' + % (option, shlex.quote(action.help)) + for action in top_level_parser._actions + for option in action.option_strings ) ) From 8060586d8b5088c2d96d91a0a34c79eb2bf8763b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 20:05:17 -0700 Subject: [PATCH 132/344] fix the script and drop unneeded options --- borgmatic/commands/completion.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index da62cccc..aadf85e2 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -64,16 +64,16 @@ def fish_completion(): borgmatic's command-line argument parsers. ''' top_level_parser, subparsers = arguments.make_parsers() - actions = ' '.join(subparsers.choices.keys()) # Avert your eyes. return '\n'.join( ( 'function __borgmatic_check_version', - ' set this_script (cat (status current-filename) 2> /dev/null)', + ' set this_filename (status current-filename)', + ' set this_script (cat $this_filename 2> /dev/null)', ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee (status current-filename)', '(status current-filename)')), + ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee $this_filename', '$this_filename')), ' end', 'end', '__borgmatic_check_version &', @@ -81,8 +81,6 @@ def fish_completion(): '''complete -c borgmatic -a '%s' -d %s -f''' % (action, shlex.quote(subparser.description)) for action, subparser in subparsers.choices.items() - ) + ( - 'complete -c borgmatic -a "%s" -d "borgmatic actions" -f' % actions, ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' % (option, shlex.quote(action.help)) From 412d18f2183292c0edd46cd8566aab0328eade50 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 21:31:53 -0700 Subject: [PATCH 133/344] show sub options --- borgmatic/commands/completion.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index aadf85e2..5555c42b 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -79,12 +79,18 @@ def fish_completion(): '__borgmatic_check_version &', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' - % (action, shlex.quote(subparser.description)) - for action, subparser in subparsers.choices.items() + % (actionStr, shlex.quote(subparser.description)) + for actionStr, subparser in subparsers.choices.items() ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' % (option, shlex.quote(action.help)) for action in top_level_parser._actions for option in action.option_strings + ) + tuple( + '''complete -c borgmatic -a '%s' -d %s -f -n "__fish_seen_subcommand_from %s"''' + % (option, shlex.quote(action.help), actionStr) + for actionStr, subparser in subparsers.choices.items() + for action in subparser._actions + for option in action.option_strings ) ) From 2e658cfa5687b90d867ef69baa8be94cc29a802d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 27 Apr 2023 21:57:50 -0700 Subject: [PATCH 134/344] only allow one parser --- borgmatic/commands/completion.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 5555c42b..225fffa7 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -65,6 +65,8 @@ def fish_completion(): ''' top_level_parser, subparsers = arguments.make_parsers() + all_subparsers = ' '.join(action for action in subparsers.choices.keys()) + # Avert your eyes. return '\n'.join( ( @@ -78,8 +80,8 @@ def fish_completion(): 'end', '__borgmatic_check_version &', ) + tuple( - '''complete -c borgmatic -a '%s' -d %s -f''' - % (actionStr, shlex.quote(subparser.description)) + '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' + % (actionStr, shlex.quote(subparser.description), all_subparsers) for actionStr, subparser in subparsers.choices.items() ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' From 77c3161c7774bec3a094da17986e395a638b215c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Apr 2023 08:36:03 -0700 Subject: [PATCH 135/344] Fix canonical home link in README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2af792ba..079ec8ae 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ borgmatic is simple, configuration-driven backup software for servers and workstations. Protect your files with client-side encryption. Backup your databases too. Monitor it all with integrated third-party services. -The canonical home of borgmatic is at https://torsion.org/borgmatic. +The canonical home of borgmatic is at https://torsion.org/borgmatic/ Here's an example configuration file: From d265b6ed6f316a16e9c945829a8da1596841f1e2 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 11:57:16 -0700 Subject: [PATCH 136/344] add comments in generated files --- borgmatic/commands/completion.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 225fffa7..56109f37 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -79,15 +79,21 @@ def fish_completion(): ' end', 'end', '__borgmatic_check_version &', + ) + ( + '# subparser completions', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' % (actionStr, shlex.quote(subparser.description), all_subparsers) for actionStr, subparser in subparsers.choices.items() + ) + ( + '# global flags', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' % (option, shlex.quote(action.help)) for action in top_level_parser._actions for option in action.option_strings + ) + ( + '# subparser flags', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "__fish_seen_subcommand_from %s"''' % (option, shlex.quote(action.help), actionStr) From 23f478ce7454fc715e23dc695abd38884854c0b8 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 12:13:08 -0700 Subject: [PATCH 137/344] use less completion lines --- borgmatic/commands/completion.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 56109f37..a256ad51 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -80,25 +80,24 @@ def fish_completion(): 'end', '__borgmatic_check_version &', ) + ( - '# subparser completions', + '\n# subparser completions', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' % (actionStr, shlex.quote(subparser.description), all_subparsers) for actionStr, subparser in subparsers.choices.items() ) + ( - '# global flags', + '\n# global flags', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' % (option, shlex.quote(action.help)) for action in top_level_parser._actions for option in action.option_strings ) + ( - '# subparser flags', + '\n# subparser flags', ) + tuple( - '''complete -c borgmatic -a '%s' -d %s -f -n "__fish_seen_subcommand_from %s"''' - % (option, shlex.quote(action.help), actionStr) + '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" -f''' + % (' '.join(action.option_strings), shlex.quote(action.help), actionStr) for actionStr, subparser in subparsers.choices.items() for action in subparser._actions - for option in action.option_strings ) ) From 9c77ebb01600b7caa093a4818b3fed662df28183 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 12:15:01 -0700 Subject: [PATCH 138/344] continue deduping --- borgmatic/commands/completion.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index a256ad51..e711b789 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -89,9 +89,8 @@ def fish_completion(): '\n# global flags', ) + tuple( '''complete -c borgmatic -a '%s' -d %s -f''' - % (option, shlex.quote(action.help)) + % (' '.join(action.option_strings), shlex.quote(action.help)) for action in top_level_parser._actions - for option in action.option_strings ) + ( '\n# subparser flags', ) + tuple( From 98e3a81fcf6d5608ce1ebb7d27fcdefc2910d6d8 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 12:42:26 -0700 Subject: [PATCH 139/344] allow file completions as applicable --- borgmatic/commands/completion.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e711b789..e62de01c 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,3 +1,4 @@ +from argparse import Action from borgmatic.commands import arguments import shlex @@ -58,6 +59,15 @@ def bash_completion(): ) ) +def build_fish_flags(action: Action): + ''' + Given an argparse.Action instance, return a string containing the fish flags for that action. + ''' + if action.metavar and action.metavar == 'PATH' or action.metavar == 'FILENAME': + return '-r -F' + else: + return '-f' + def fish_completion(): ''' Return a fish completion script for the borgmatic command. Produce this by introspecting @@ -88,14 +98,14 @@ def fish_completion(): ) + ( '\n# global flags', ) + tuple( - '''complete -c borgmatic -a '%s' -d %s -f''' - % (' '.join(action.option_strings), shlex.quote(action.help)) + '''complete -c borgmatic -a '%s' -d %s %s''' + % (' '.join(action.option_strings), shlex.quote(action.help), build_fish_flags(action)) for action in top_level_parser._actions ) + ( '\n# subparser flags', ) + tuple( - '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" -f''' - % (' '.join(action.option_strings), shlex.quote(action.help), actionStr) + '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" %s''' + % (' '.join(action.option_strings), shlex.quote(action.help), actionStr, build_fish_flags(action)) for actionStr, subparser in subparsers.choices.items() for action in subparser._actions ) From f7e4024fcae5fc1aa836f0cb90f379df65cdb251 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 28 Apr 2023 14:02:06 -0700 Subject: [PATCH 140/344] add to readme --- docs/how-to/set-up-backups.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index be229cb2..098f0139 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -334,10 +334,13 @@ Access](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/293). ### Shell completion -borgmatic includes a shell completion script (currently only for Bash) to +borgmatic includes a shell completion script (currently only for Bash and Fish) to support tab-completing borgmatic command-line actions and flags. Depending on -how you installed borgmatic, this may be enabled by default. But if it's not, -start by installing the `bash-completion` Linux package or the +how you installed borgmatic, this may be enabled by default. + +#### Bash + +If completions aren't enabled, start by installing the `bash-completion` Linux package or the [`bash-completion@2`](https://formulae.brew.sh/formula/bash-completion@2) macOS Homebrew formula. Then, install the shell completion script globally: @@ -362,6 +365,14 @@ borgmatic --bash-completion > ~/.local/share/bash-completion/completions/borgmat Finally, restart your shell (`exit` and open a new shell) so the completions take effect. +#### Fish + +To add completions for fish, install the completions file globally: + +```fish +borgmatic --fish-completion | sudo tee /usr/share/fish/vendor_completions.d/borgmatic.fish +source /usr/share/fish/vendor_completions.d/borgmatic.fish +``` ### Colored output From a60d7fd173a4fa045ac3b17bae864269cb88a59a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 30 Apr 2023 15:43:41 -0700 Subject: [PATCH 141/344] Run "borgmatic borg" action without capturing output so interactive prompts and flags like "--progress" still work. --- NEWS | 2 ++ borgmatic/borg/borg.py | 4 ++-- docs/how-to/monitor-your-backups.md | 2 +- docs/how-to/run-arbitrary-borg-commands.md | 6 +++-- tests/unit/borg/test_borg.py | 28 +++++++++++----------- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/NEWS b/NEWS index 20214674..e196a70f 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,8 @@ * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. + * Run "borgmatic borg" action without capturing output so interactive prompts and flags like + "--progress" still work. 1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index f19d6555..1c41b8ec 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -2,7 +2,7 @@ import logging import borgmatic.logger from borgmatic.borg import environment, flags -from borgmatic.execute import execute_command +from borgmatic.execute import DO_NOT_CAPTURE, execute_command logger = logging.getLogger(__name__) @@ -62,7 +62,7 @@ def run_arbitrary_borg( return execute_command( full_command, - output_log_level=logging.ANSWER, + output_file=DO_NOT_CAPTURE, borg_local_path=local_path, extra_environment=environment.make_environment(storage_config), ) diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index eb7a6200..517f9c79 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -73,7 +73,7 @@ from borgmatic for a configured interval. ### Consistency checks -While not strictly part of monitoring, if you really want confidence that your +While not strictly part of monitoring, if you want confidence that your backups are not only running but are restorable as well, you can configure particular [consistency checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index 0777ebba..ea265eaa 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -89,8 +89,10 @@ borgmatic's `borg` action is not without limitations: * Unlike normal borgmatic actions that support JSON, the `borg` action will not disable certain borgmatic logs to avoid interfering with JSON output. * Unlike other borgmatic actions, the `borg` action captures (and logs) all - output, so interactive prompts or flags like `--progress` will not work as - expected. + output, so interactive prompts and flags like `--progress` will not work as + expected. New in version + 1.7.13 borgmatic now runs the `borg` action without capturing output, + so interactive prompts work. In general, this `borgmatic borg` feature should be considered an escape valve—a feature of second resort. In the long run, it's preferable to wrap diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 5b735960..4c71ce1a 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -15,7 +15,7 @@ def test_run_arbitrary_borg_calls_borg_with_parameters(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -36,7 +36,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--info'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -58,7 +58,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--debug', '--show-rc'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -83,7 +83,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters( flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--lock-wait', '5'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -106,7 +106,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo::archive'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -128,7 +128,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg1', 'break-lock', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg1', extra_environment=None, ) @@ -152,7 +152,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--remote-path', 'borg1'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -174,7 +174,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo', '--progress'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -195,7 +195,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -216,7 +216,7 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg',), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -237,7 +237,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'key', 'export', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -258,7 +258,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository() flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'debug', 'dump-manifest', 'repo', 'path'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -279,7 +279,7 @@ def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repositor flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'debug', 'info'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -300,7 +300,7 @@ def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_bor flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'debug', 'convert-profile', 'in', 'out'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) From 0b397a5bf93e5255f515a59ea49d091db51e90d9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 30 Apr 2023 16:24:10 -0700 Subject: [PATCH 142/344] Fix borgmatic error when not finding the configuration schema for certain "pip install --editable" development installs (#687). --- NEWS | 2 ++ borgmatic/config/validate.py | 5 ++++- tests/unit/config/test_validate.py | 5 ++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index e196a70f..30a2b9b0 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,8 @@ * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. + * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install + --editable" development installs. * Run "borgmatic borg" action without capturing output so interactive prompts and flags like "--progress" still work. diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 537f4bee..b39199fe 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -8,6 +8,7 @@ try: except ModuleNotFoundError: # pragma: nocover import importlib.metadata as importlib_metadata +import borgmatic.config from borgmatic.config import environment, load, normalize, override @@ -25,7 +26,9 @@ def schema_filename(): if path.match('config/schema.yaml') ) except StopIteration: - raise FileNotFoundError('Configuration file schema could not be found') + # If the schema wasn't found in the package's files, this is probably a pip editable + # install, so try a different approach to get the schema. + return os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml') def format_json_error_path_element(path_element): diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index e2b9f98f..e81f2b02 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -16,14 +16,13 @@ def test_schema_filename_finds_schema_path(): assert module.schema_filename() == schema_path -def test_schema_filename_with_missing_schema_path_raises(): +def test_schema_filename_with_missing_schema_path_in_package_still_finds_it_in_config_directory(): flexmock(module.importlib_metadata).should_receive('files').and_return( flexmock(match=lambda path: False, locate=lambda: None), flexmock(match=lambda path: False, locate=lambda: None), ) - with pytest.raises(FileNotFoundError): - assert module.schema_filename() + assert module.schema_filename().endswith('/borgmatic/config/schema.yaml') def test_format_json_error_path_element_formats_array_index(): From 359afe531803938205a648f8f1093343c2360f16 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 3 May 2023 17:16:36 -0700 Subject: [PATCH 143/344] Error if --list is used with --json for create action (#680). --- borgmatic/commands/arguments.py | 4 ++++ tests/integration/commands/test_arguments.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 61b54769..3a8ef2e2 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -878,6 +878,10 @@ def parse_arguments(*unparsed_arguments): raise ValueError( 'With the create action, only one of --list (--files) and --progress flags can be used.' ) + if 'create' in arguments and arguments['create'].list_files and arguments['create'].json: + raise ValueError( + 'With the create action, only one of --list (--files) and --json flags can be used.' + ) if ( ('list' in arguments and 'rinfo' in arguments and arguments['list'].json) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 1fc8f8c8..4990fc4f 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -422,6 +422,13 @@ def test_parse_arguments_disallows_list_with_progress_for_create_action(): module.parse_arguments('create', '--list', '--progress') +def test_parse_arguments_disallows_list_with_json_for_create_action(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('create', '--list', '--json') + + def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) From 9ff5ea52409b8e426600996f9a398fcb386aee88 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:20:01 -0700 Subject: [PATCH 144/344] add a unit test, fix isort and black --- borgmatic/commands/completion.py | 49 +++++++++++++------ tests/integration/commands/test_completion.py | 4 ++ 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e62de01c..352dc365 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,6 +1,8 @@ -from argparse import Action -from borgmatic.commands import arguments import shlex +from argparse import Action + +from borgmatic.commands import arguments + def upgrade_message(language: str, upgrade_command: str, completion_file: str): return f''' @@ -37,7 +39,13 @@ def bash_completion(): ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - ' then cat << EOF\n{}\nEOF'.format(upgrade_message('bash', 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', '$BASH_SOURCE')), + ' then cat << EOF\n{}\nEOF'.format( + upgrade_message( + 'bash', + 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', + '$BASH_SOURCE', + ) + ), ' fi', '}', 'complete_borgmatic() {', @@ -59,6 +67,7 @@ def bash_completion(): ) ) + def build_fish_flags(action: Action): ''' Given an argparse.Action instance, return a string containing the fish flags for that action. @@ -68,6 +77,7 @@ def build_fish_flags(action: Action): else: return '-f' + def fish_completion(): ''' Return a fish completion script for the borgmatic command. Produce this by introspecting @@ -85,27 +95,38 @@ def fish_completion(): ' set this_script (cat $this_filename 2> /dev/null)', ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - ' echo "{}"'.format(upgrade_message('fish', 'borgmatic --fish-completion | sudo tee $this_filename', '$this_filename')), + ' echo "{}"'.format( + upgrade_message( + 'fish', + 'borgmatic --fish-completion | sudo tee $this_filename', + '$this_filename', + ) + ), ' end', 'end', '__borgmatic_check_version &', - ) + ( - '\n# subparser completions', - ) + tuple( + ) + + ('\n# subparser completions',) + + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' % (actionStr, shlex.quote(subparser.description), all_subparsers) for actionStr, subparser in subparsers.choices.items() - ) + ( - '\n# global flags', - ) + tuple( + ) + + ('\n# global flags',) + + tuple( '''complete -c borgmatic -a '%s' -d %s %s''' % (' '.join(action.option_strings), shlex.quote(action.help), build_fish_flags(action)) for action in top_level_parser._actions - ) + ( - '\n# subparser flags', - ) + tuple( + ) + + ('\n# subparser flags',) + + tuple( '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" %s''' - % (' '.join(action.option_strings), shlex.quote(action.help), actionStr, build_fish_flags(action)) + % ( + ' '.join(action.option_strings), + shlex.quote(action.help), + actionStr, + build_fish_flags(action), + ) for actionStr, subparser in subparsers.choices.items() for action in subparser._actions ) diff --git a/tests/integration/commands/test_completion.py b/tests/integration/commands/test_completion.py index a3b0b9c2..9a118abf 100644 --- a/tests/integration/commands/test_completion.py +++ b/tests/integration/commands/test_completion.py @@ -3,3 +3,7 @@ from borgmatic.commands import completion as module def test_bash_completion_does_not_raise(): assert module.bash_completion() + + +def test_fish_completion_does_not_raise(): + assert module.fish_completion() From ca689505e57261fb35c6229014b6a0a7e089c87c Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:27:00 -0700 Subject: [PATCH 145/344] add e2e fish test --- scripts/run-full-tests | 6 +++--- tests/end-to-end/test_completion.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/run-full-tests b/scripts/run-full-tests index bf26c212..a7a49a2a 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -10,7 +10,7 @@ set -e -if [ -z "$TEST_CONTAINER" ] ; then +if [ -z "$TEST_CONTAINER" ]; then echo "This script is designed to work inside a test container and is not intended to" echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute" echo "scripts/run-end-to-end-dev-tests instead." @@ -18,14 +18,14 @@ if [ -z "$TEST_CONTAINER" ] ; then fi apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \ - py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite + py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite fish # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 pip3 install --ignore-installed tox==3.25.1 export COVERAGE_FILE=/tmp/.coverage -if [ "$1" != "--end-to-end-only" ] ; then +if [ "$1" != "--end-to-end-only" ]; then tox --workdir /tmp/.tox --sitepackages fi diff --git a/tests/end-to-end/test_completion.py b/tests/end-to-end/test_completion.py index e4037ece..7d6af4ce 100644 --- a/tests/end-to-end/test_completion.py +++ b/tests/end-to-end/test_completion.py @@ -3,3 +3,7 @@ import subprocess def test_bash_completion_runs_without_error(): subprocess.check_call('borgmatic --bash-completion | bash', shell=True) + + +def test_fish_completion_runs_without_error(): + subprocess.check_call('borgmatic --fish-completion | fish', shell=True) From b7fe2a503173614361cd94e946647790f57ec7a5 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:27:57 -0700 Subject: [PATCH 146/344] lowercase fish in docs --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 098f0139..de5bf8b9 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -365,7 +365,7 @@ borgmatic --bash-completion > ~/.local/share/bash-completion/completions/borgmat Finally, restart your shell (`exit` and open a new shell) so the completions take effect. -#### Fish +#### fish To add completions for fish, install the completions file globally: From 062453af51b7c99401e1508f989a116d01b20a09 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:29:25 -0700 Subject: [PATCH 147/344] replace actionStr with action_name --- borgmatic/commands/completion.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 352dc365..92a221d4 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -109,8 +109,8 @@ def fish_completion(): + ('\n# subparser completions',) + tuple( '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' - % (actionStr, shlex.quote(subparser.description), all_subparsers) - for actionStr, subparser in subparsers.choices.items() + % (action_name, shlex.quote(subparser.description), all_subparsers) + for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) + tuple( @@ -124,10 +124,10 @@ def fish_completion(): % ( ' '.join(action.option_strings), shlex.quote(action.help), - actionStr, + action_name, build_fish_flags(action), ) - for actionStr, subparser in subparsers.choices.items() + for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) ) From f04036e4a76dc99b4764f33b067f2d9c7f5f0fbc Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:33:21 -0700 Subject: [PATCH 148/344] use fstring to produce completion lines --- borgmatic/commands/completion.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 92a221d4..6a686d2b 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -108,25 +108,17 @@ def fish_completion(): ) + ('\n# subparser completions',) + tuple( - '''complete -c borgmatic -a '%s' -d %s -f -n "not __fish_seen_subcommand_from %s"''' - % (action_name, shlex.quote(subparser.description), all_subparsers) + f'''complete -c borgmatic -a '{action_name}' -d {shlex.quote(subparser.description)} -f -n "not __fish_seen_subcommand_from {all_subparsers}"''' for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) + tuple( - '''complete -c borgmatic -a '%s' -d %s %s''' - % (' '.join(action.option_strings), shlex.quote(action.help), build_fish_flags(action)) + f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} {build_fish_flags(action)}''' for action in top_level_parser._actions ) + ('\n# subparser flags',) + tuple( - '''complete -c borgmatic -a '%s' -d %s -n "__fish_seen_subcommand_from %s" %s''' - % ( - ' '.join(action.option_strings), - shlex.quote(action.help), - action_name, - build_fish_flags(action), - ) + f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}" {build_fish_flags(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) From 700f8e9d9c1ccd7e67a05a0d63647245a057272b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:39:48 -0700 Subject: [PATCH 149/344] replace .format with fstring --- borgmatic/commands/completion.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 6a686d2b..e206df73 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -39,13 +39,11 @@ def bash_completion(): ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - ' then cat << EOF\n{}\nEOF'.format( - upgrade_message( + f''' then cat << EOF\n{upgrade_message( 'bash', 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', '$BASH_SOURCE', - ) - ), + )}\nEOF''', ' fi', '}', 'complete_borgmatic() {', @@ -95,13 +93,11 @@ def fish_completion(): ' set this_script (cat $this_filename 2> /dev/null)', ' set installed_script (borgmatic --fish-completion 2> /dev/null)', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - ' echo "{}"'.format( - upgrade_message( + f''' echo "{upgrade_message( 'fish', 'borgmatic --fish-completion | sudo tee $this_filename', '$this_filename', - ) - ), + )}"''', ' end', 'end', '__borgmatic_check_version &', From f1fd2e88dd5d947341e37bbe2352ac1399ab5151 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 13:49:29 -0700 Subject: [PATCH 150/344] drop blank completion --- borgmatic/commands/completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e206df73..ec4d8d15 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -111,6 +111,7 @@ def fish_completion(): + tuple( f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} {build_fish_flags(action)}''' for action in top_level_parser._actions + if len(action.option_strings) > 0 ) + ('\n# subparser flags',) + tuple( From 28efc8566075a53eff78783129d26111b0b804d8 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 18:11:13 -0700 Subject: [PATCH 151/344] rearrange to improve legability of the file --- borgmatic/commands/completion.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index ec4d8d15..c8696e60 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -102,9 +102,10 @@ def fish_completion(): 'end', '__borgmatic_check_version &', ) + + (f'''set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}"''',) + ('\n# subparser completions',) + tuple( - f'''complete -c borgmatic -a '{action_name}' -d {shlex.quote(subparser.description)} -f -n "not __fish_seen_subcommand_from {all_subparsers}"''' + f'''complete -c borgmatic -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)} -f''' for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) From f12a10d888c2cdc5b5eb581dc40938605fb09fe2 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 19:50:49 -0700 Subject: [PATCH 152/344] start work on conditional file completion --- borgmatic/commands/completion.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index c8696e60..5ae7a30a 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,5 +1,6 @@ import shlex from argparse import Action +from textwrap import dedent from borgmatic.commands import arguments @@ -66,14 +67,20 @@ def bash_completion(): ) -def build_fish_flags(action: Action): +def conditionally_emit_file_completion(action: Action): ''' - Given an argparse.Action instance, return a string containing the fish flags for that action. + Given an argparse.Action instance, return a completion invocation that forces file completion + if the action takes a file argument and was the last action on the command line. + + Otherwise, return an empty string. ''' - if action.metavar and action.metavar == 'PATH' or action.metavar == 'FILENAME': - return '-r -F' - else: - return '-f' + if not action.metavar: + return '' + + args = ' '.join(action.option_strings) + + return dedent(f''' + complete -c borgmatic -a {args} -Fr -n "__borgmatic_last_arg {args}"''') def fish_completion(): @@ -105,18 +112,18 @@ def fish_completion(): + (f'''set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}"''',) + ('\n# subparser completions',) + tuple( - f'''complete -c borgmatic -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)} -f''' + f'''complete -c borgmatic -f -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) + tuple( - f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} {build_fish_flags(action)}''' + f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{conditionally_emit_file_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 ) + ('\n# subparser flags',) + tuple( - f'''complete -c borgmatic -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}" {build_fish_flags(action)}''' + f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{conditionally_emit_file_completion(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) From 639e88262e3d0f7a27fdb7359bec44f9fa22d318 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 20:17:26 -0700 Subject: [PATCH 153/344] create working file completion --- borgmatic/commands/completion.py | 56 +++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 5ae7a30a..9a21334d 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -76,11 +76,17 @@ def conditionally_emit_file_completion(action: Action): ''' if not action.metavar: return '' - + args = ' '.join(action.option_strings) - - return dedent(f''' - complete -c borgmatic -a {args} -Fr -n "__borgmatic_last_arg {args}"''') + + return dedent( + f''' + complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' + ) + + +def dedent_strip_as_tuple(string: str): + return (dedent(string).strip("\n"),) def fish_completion(): @@ -94,22 +100,40 @@ def fish_completion(): # Avert your eyes. return '\n'.join( - ( - 'function __borgmatic_check_version', - ' set this_filename (status current-filename)', - ' set this_script (cat $this_filename 2> /dev/null)', - ' set installed_script (borgmatic --fish-completion 2> /dev/null)', - ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]', - f''' echo "{upgrade_message( + dedent_strip_as_tuple( + f''' + function __borgmatic_check_version + set this_filename (status current-filename) + set this_script (cat $this_filename 2> /dev/null) + set installed_script (borgmatic --fish-completion 2> /dev/null) + if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] + echo "{upgrade_message( 'fish', 'borgmatic --fish-completion | sudo tee $this_filename', '$this_filename', - )}"''', - ' end', - 'end', - '__borgmatic_check_version &', + )}" + end + end + __borgmatic_check_version & + + function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line' + set -l all_args (commandline -poc) + # premature optimization to avoid iterating all args if there aren't enough + # to have a last arg beyond borgmatic + if [ (count $all_args) -lt 2 ] + return 1 + end + for arg in $argv + if [ "$arg" = "$all_args[-1]" ] + return 0 + end + end + return 1 + end + + set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}" + ''' ) - + (f'''set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}"''',) + ('\n# subparser completions',) + tuple( f'''complete -c borgmatic -f -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' From bbc3e9d7173784058646ee3c8325b976a2b04034 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 21:12:24 -0700 Subject: [PATCH 154/344] show possible choices --- borgmatic/commands/completion.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 9a21334d..fb82e2a4 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -66,6 +66,15 @@ def bash_completion(): ) ) +file_metavars = ( + 'FILENAME', + 'PATH', +) + +file_destinations = ( + 'config_paths' +) + def conditionally_emit_file_completion(action: Action): ''' @@ -74,15 +83,22 @@ def conditionally_emit_file_completion(action: Action): Otherwise, return an empty string. ''' - if not action.metavar: - return '' args = ' '.join(action.option_strings) - return dedent( - f''' - complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' - ) + if action.metavar in file_metavars or action.dest in file_destinations: + return dedent( + f''' + complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' + ) + + if action.choices: + return dedent( + f''' + complete -c borgmatic -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' + ) + + return '' def dedent_strip_as_tuple(string: str): From 193731a017a87b267f126fe4fb02a0bc52eff36d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 21:14:48 -0700 Subject: [PATCH 155/344] rename function --- borgmatic/commands/completion.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index fb82e2a4..be94dd0f 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -66,20 +66,20 @@ def bash_completion(): ) ) + file_metavars = ( 'FILENAME', 'PATH', ) -file_destinations = ( - 'config_paths' -) +file_destinations = 'config_paths' -def conditionally_emit_file_completion(action: Action): +def conditionally_emit_arg_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation that forces file completion - if the action takes a file argument and was the last action on the command line. + Given an argparse.Action instance, return a completion invocation + that forces file completion or options completion, if the action + takes such an argument and was the last action on the command line. Otherwise, return an empty string. ''' @@ -91,7 +91,7 @@ def conditionally_emit_file_completion(action: Action): f''' complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' ) - + if action.choices: return dedent( f''' @@ -157,13 +157,13 @@ def fish_completion(): ) + ('\n# global flags',) + tuple( - f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{conditionally_emit_file_completion(action)}''' + f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{conditionally_emit_arg_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 ) + ('\n# subparser flags',) + tuple( - f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{conditionally_emit_file_completion(action)}''' + f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{conditionally_emit_arg_completion(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) From d962376a9dd3b1c643bbe3c22de88d329eafe577 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 21:58:30 -0700 Subject: [PATCH 156/344] refactor to only show specific options if possible --- borgmatic/commands/completion.py | 42 ++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index be94dd0f..0b1deb4f 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -75,7 +75,11 @@ file_metavars = ( file_destinations = 'config_paths' -def conditionally_emit_arg_completion(action: Action): +def has_exact_options(action: Action): + return action.metavar in file_metavars or action.dest in file_destinations or action.choices + + +def exact_options_completion(action: Action): ''' Given an argparse.Action instance, return a completion invocation that forces file completion or options completion, if the action @@ -84,21 +88,20 @@ def conditionally_emit_arg_completion(action: Action): Otherwise, return an empty string. ''' + if not has_exact_options(action): + return '' + args = ' '.join(action.option_strings) if action.metavar in file_metavars or action.dest in file_destinations: - return dedent( - f''' - complete -c borgmatic -a '{args}' -Fr -n "__borgmatic_last_arg {args}"''' - ) + return f'''\ncomplete -c borgmatic -Fr -a '{args}' -n "__borgmatic_last_arg {args}"''' if action.choices: - return dedent( - f''' - complete -c borgmatic -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' - ) + return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' - return '' + raise RuntimeError( + f'Unexpected action: {action} passes has_exact_options but has no choices produced' + ) def dedent_strip_as_tuple(string: str): @@ -114,6 +117,18 @@ def fish_completion(): all_subparsers = ' '.join(action for action in subparsers.choices.keys()) + exact_option_args = tuple( + ' '.join(action.option_strings) + for subparser in subparsers.choices.values() + for action in subparser._actions + if has_exact_options(action) + ) + tuple( + ' '.join(action.option_strings) + for action in top_level_parser._actions + if len(action.option_strings) > 0 + if has_exact_options(action) + ) + # Avert your eyes. return '\n'.join( dedent_strip_as_tuple( @@ -148,22 +163,23 @@ def fish_completion(): end set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}" + set --local exact_option_condition "not __borgmatic_last_arg {' '.join(exact_option_args)}" ''' ) + ('\n# subparser completions',) + tuple( - f'''complete -c borgmatic -f -n "$subparser_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' + f'''complete -c borgmatic -f -n "$subparser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' for action_name, subparser in subparsers.choices.items() ) + ('\n# global flags',) + tuple( - f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{conditionally_emit_arg_completion(action)}''' + f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 ) + ('\n# subparser flags',) + tuple( - f'''complete -c borgmatic -f -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{conditionally_emit_arg_completion(action)}''' + f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions ) From b4a38d8be9428a49eb3d47ef3fbda896d81eed95 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:06:11 -0700 Subject: [PATCH 157/344] fix flag showing up for paths --- borgmatic/commands/completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 0b1deb4f..0f6c3c20 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -94,7 +94,7 @@ def exact_options_completion(action: Action): args = ' '.join(action.option_strings) if action.metavar in file_metavars or action.dest in file_destinations: - return f'''\ncomplete -c borgmatic -Fr -a '{args}' -n "__borgmatic_last_arg {args}"''' + return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_last_arg {args}"''' if action.choices: return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' From 8f3039be2332c348810f330ede2466b529db0290 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:23:29 -0700 Subject: [PATCH 158/344] handle the expanding filters better --- borgmatic/commands/completion.py | 38 +++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 0f6c3c20..c22e50fd 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -67,16 +67,35 @@ def bash_completion(): ) -file_metavars = ( - 'FILENAME', - 'PATH', -) +# fish section -file_destinations = 'config_paths' +def has_file_options(action: Action): + return action.metavar in ( + 'FILENAME', + 'PATH', + ) or action.dest in ('config_paths',) + + +def has_choice_options(action: Action): + return action.choices is not None + + +def has_required_param_options(action: Action): + return ( + action.nargs + in ( + "+", + "*", + ) + or '--archive' in action.option_strings + or action.metavar in ('PATTERN', 'KEYS', 'N') + ) def has_exact_options(action: Action): - return action.metavar in file_metavars or action.dest in file_destinations or action.choices + return ( + has_file_options(action) or has_choice_options(action) or has_required_param_options(action) + ) def exact_options_completion(action: Action): @@ -93,12 +112,15 @@ def exact_options_completion(action: Action): args = ' '.join(action.option_strings) - if action.metavar in file_metavars or action.dest in file_destinations: + if has_file_options(action): return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_last_arg {args}"''' - if action.choices: + if has_choice_options(action): return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' + if has_required_param_options(action): + return f'''\ncomplete -c borgmatic -x -n "__borgmatic_last_arg {args}"''' + raise RuntimeError( f'Unexpected action: {action} passes has_exact_options but has no choices produced' ) From 3592ec3ddf44dd45f256d25b239bf0450a03bd99 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:32:09 -0700 Subject: [PATCH 159/344] dont show deprecated options --- borgmatic/commands/completion.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index c22e50fd..eb45799f 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -198,11 +198,13 @@ def fish_completion(): f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 + if 'Deprecated' not in action.help ) + ('\n# subparser flags',) + tuple( f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions + if 'Deprecated' not in action.help ) ) From 16ac4824a51965d55162044e43edcba8f01b6102 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:42:04 -0700 Subject: [PATCH 160/344] handle typed without default params --- borgmatic/commands/completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index eb45799f..d24c0430 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -89,6 +89,7 @@ def has_required_param_options(action: Action): ) or '--archive' in action.option_strings or action.metavar in ('PATTERN', 'KEYS', 'N') + or (action.type is not None and action.default is None) ) From d59b9b817f3ebf8ab4f60b664c6fbbfbc49da9fd Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:44:54 -0700 Subject: [PATCH 161/344] support required actions --- borgmatic/commands/completion.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index d24c0430..ee83c28e 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -69,6 +69,7 @@ def bash_completion(): # fish section + def has_file_options(action: Action): return action.metavar in ( 'FILENAME', @@ -82,7 +83,8 @@ def has_choice_options(action: Action): def has_required_param_options(action: Action): return ( - action.nargs + action.required is True + or action.nargs in ( "+", "*", From b557d635fd18c6c952ff26432f396e9f6cfef37d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 May 2023 23:57:37 -0700 Subject: [PATCH 162/344] async validity check --- borgmatic/commands/completion.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index ee83c28e..2dc63e67 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -159,18 +159,20 @@ def fish_completion(): dedent_strip_as_tuple( f''' function __borgmatic_check_version - set this_filename (status current-filename) - set this_script (cat $this_filename 2> /dev/null) - set installed_script (borgmatic --fish-completion 2> /dev/null) - if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] - echo "{upgrade_message( - 'fish', - 'borgmatic --fish-completion | sudo tee $this_filename', - '$this_filename', - )}" - end + set -fx this_filename (status current-filename) + fish -c ' + set this_script (cat $this_filename 2> /dev/null) + set installed_script (borgmatic --fish-completion 2> /dev/null) + if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] + echo "{upgrade_message( + 'fish', + 'borgmatic --fish-completion | sudo tee $this_filename', + '$this_filename', + )}" + end + ' & end - __borgmatic_check_version & + __borgmatic_check_version function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line' set -l all_args (commandline -poc) From 5a7a1747f29a16e8b3c2508ee0dcf2def5c58b85 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 5 May 2023 00:01:45 -0700 Subject: [PATCH 163/344] add safety check to avoid infinite cat hang --- borgmatic/commands/completion.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 2dc63e67..3ce5c03e 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -161,14 +161,16 @@ def fish_completion(): function __borgmatic_check_version set -fx this_filename (status current-filename) fish -c ' - set this_script (cat $this_filename 2> /dev/null) - set installed_script (borgmatic --fish-completion 2> /dev/null) - if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] - echo "{upgrade_message( - 'fish', - 'borgmatic --fish-completion | sudo tee $this_filename', - '$this_filename', - )}" + if test -f "$this_filename" + set this_script (cat $this_filename 2> /dev/null) + set installed_script (borgmatic --fish-completion 2> /dev/null) + if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] + echo "{upgrade_message( + 'fish', + 'borgmatic --fish-completion | sudo tee $this_filename', + '$this_filename', + )}" + end end ' & end From 59a6ce1462d87dea5ebabbecf4bb4e18f3e80d44 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 5 May 2023 00:03:43 -0700 Subject: [PATCH 164/344] replace double quotes with single quotes --- borgmatic/commands/completion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 3ce5c03e..e99f3903 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -86,8 +86,8 @@ def has_required_param_options(action: Action): action.required is True or action.nargs in ( - "+", - "*", + '+', + '*', ) or '--archive' in action.option_strings or action.metavar in ('PATTERN', 'KEYS', 'N') @@ -130,7 +130,7 @@ def exact_options_completion(action: Action): def dedent_strip_as_tuple(string: str): - return (dedent(string).strip("\n"),) + return (dedent(string).strip('\n'),) def fish_completion(): From 469e0ccace89270ff3763614ad125ccacd956702 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 10:42:06 -0700 Subject: [PATCH 165/344] create doccomments, start writing unit tests --- borgmatic/commands/completion.py | 11 ++++++++--- tests/unit/commands/test_completions.py | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 tests/unit/commands/test_completions.py diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e99f3903..e160a8fc 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -71,6 +71,9 @@ def bash_completion(): def has_file_options(action: Action): + ''' + Given an argparse.Action instance, return True if it takes a file argument. + ''' return action.metavar in ( 'FILENAME', 'PATH', @@ -78,6 +81,9 @@ def has_file_options(action: Action): def has_choice_options(action: Action): + ''' + Given an argparse.Action instance, return True if it takes one of a predefined set of arguments. + ''' return action.choices is not None @@ -103,9 +109,8 @@ def has_exact_options(action: Action): def exact_options_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation - that forces file completion or options completion, if the action - takes such an argument and was the last action on the command line. + Given an argparse.Action instance, return a completion invocation that forces file completion or options + completion, if the action takes such an argument and was the last action on the command line. Otherwise, return an empty string. ''' diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py new file mode 100644 index 00000000..4cc1f456 --- /dev/null +++ b/tests/unit/commands/test_completions.py @@ -0,0 +1,21 @@ +from argparse import Action + +import pytest + +from borgmatic.commands.completion import has_exact_options, has_file_options + +file_options_test_data = [ + (Action('--flag', 'flag'), False), + (Action('--flag', 'flag', metavar='FILENAME'), True), + (Action('--flag', 'flag', metavar='PATH'), True), + (Action('--flag', dest='config_paths'), True), + (Action('--flag', 'flag', metavar='OTHER'), False), +] + + +@pytest.mark.parametrize('action, expected', file_options_test_data) +def test_has_file_options_detects_file_options(action: Action, expected: bool): + assert has_file_options(action) == expected + # if has_file_options(action) was true, has_exact_options(action) should also be true + if expected: + assert has_exact_options(action) From 372622fbb107c4bbfee99c4e4f0d57e3e3a6e281 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 10:46:27 -0700 Subject: [PATCH 166/344] add more doccomments, drop a check --- borgmatic/commands/completion.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e160a8fc..702a09dd 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -87,7 +87,13 @@ def has_choice_options(action: Action): return action.choices is not None -def has_required_param_options(action: Action): +def has_unknown_required_param_options(action: Action): + ''' + A catch-all for options that take a required parameter, but we don't know what the parameter is. + This should be used last. These are actions that take something like a glob, a list of numbers, or a string. + There is no way to know what the valid options are, but we need to prevent another argument from being shown, + and let the user know that they need to provide a parameter. + ''' return ( action.required is True or action.nargs @@ -95,7 +101,6 @@ def has_required_param_options(action: Action): '+', '*', ) - or '--archive' in action.option_strings or action.metavar in ('PATTERN', 'KEYS', 'N') or (action.type is not None and action.default is None) ) @@ -103,7 +108,9 @@ def has_required_param_options(action: Action): def has_exact_options(action: Action): return ( - has_file_options(action) or has_choice_options(action) or has_required_param_options(action) + has_file_options(action) + or has_choice_options(action) + or has_unknown_required_param_options(action) ) @@ -126,7 +133,7 @@ def exact_options_completion(action: Action): if has_choice_options(action): return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' - if has_required_param_options(action): + if has_unknown_required_param_options(action): return f'''\ncomplete -c borgmatic -x -n "__borgmatic_last_arg {args}"''' raise RuntimeError( From e623f401b90031c7bcb948cad9f25574c614196b Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 10:56:54 -0700 Subject: [PATCH 167/344] write more unit tests --- tests/unit/commands/test_completions.py | 42 ++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 4cc1f456..4de34fe7 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -2,7 +2,12 @@ from argparse import Action import pytest -from borgmatic.commands.completion import has_exact_options, has_file_options +from borgmatic.commands.completion import ( + has_choice_options, + has_exact_options, + has_file_options, + has_unknown_required_param_options, +) file_options_test_data = [ (Action('--flag', 'flag'), False), @@ -19,3 +24,38 @@ def test_has_file_options_detects_file_options(action: Action, expected: bool): # if has_file_options(action) was true, has_exact_options(action) should also be true if expected: assert has_exact_options(action) + + +choices_test_data = [ + (Action('--flag', 'flag'), False), + (Action('--flag', 'flag', choices=['a', 'b']), True), + (Action('--flag', 'flag', choices=None), False), +] + + +@pytest.mark.parametrize('action, expected', choices_test_data) +def test_has_choice_options_detects_choice_options(action: Action, expected: bool): + assert has_choice_options(action) == expected + # if has_choice_options(action) was true, has_exact_options(action) should also be true + if expected: + assert has_exact_options(action) + + +unknown_required_param_test_data = [ + (Action('--flag', 'flag'), False), + (Action('--flag', 'flag', required=True), True), + *((Action('--flag', 'flag', nargs=nargs), True) for nargs in ('+', '*')), + *((Action('--flag', 'flag', metavar=metavar), True) for metavar in ('PATTERN', 'KEYS', 'N')), + *((Action('--flag', 'flag', type=type, default=None), True) for type in (int, str)), + (Action('--flag', 'flag', type=int, default=1), False), +] + + +@pytest.mark.parametrize('action, expected', unknown_required_param_test_data) +def test_has_unknown_required_param_options_detects_unknown_required_param_options( + action: Action, expected: bool +): + assert has_unknown_required_param_options(action) == expected + # if has_unknown_required_param_options(action) was true, has_exact_options(action) should also be true + if expected: + assert has_exact_options(action) From 77dbb5c499d19ca40f98f68d8f91acb9c3939e8a Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:16:45 -0700 Subject: [PATCH 168/344] create way for test cases to be shared --- tests/unit/commands/test_completions.py | 129 +++++++++++++++++------- 1 file changed, 94 insertions(+), 35 deletions(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 4de34fe7..51ab29ac 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -1,4 +1,6 @@ from argparse import Action +from collections import namedtuple +from typing import Tuple import pytest @@ -9,53 +11,110 @@ from borgmatic.commands.completion import ( has_unknown_required_param_options, ) -file_options_test_data = [ - (Action('--flag', 'flag'), False), - (Action('--flag', 'flag', metavar='FILENAME'), True), - (Action('--flag', 'flag', metavar='PATH'), True), - (Action('--flag', dest='config_paths'), True), - (Action('--flag', 'flag', metavar='OTHER'), False), +OptionType = namedtuple('OptionType', ['file', 'choice', 'unknown_required']) +TestCase = Tuple[Action, OptionType] + +test_data: list[TestCase] = [ + (Action('--flag', 'flag'), OptionType(file=False, choice=False, unknown_required=False)), + *( + ( + Action('--flag', 'flag', metavar=metavar), + OptionType(file=True, choice=False, unknown_required=False), + ) + for metavar in ('FILENAME', 'PATH') + ), + ( + Action('--flag', dest='config_paths'), + OptionType(file=True, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', metavar='OTHER'), + OptionType(file=False, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', choices=['a', 'b']), + OptionType(file=False, choice=True, unknown_required=False), + ), + ( + Action('--flag', 'flag', choices=['a', 'b'], type=str), + OptionType(file=False, choice=True, unknown_required=True), + ), + ( + Action('--flag', 'flag', choices=None), + OptionType(file=False, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', required=True), + OptionType(file=False, choice=False, unknown_required=True), + ), + *( + ( + Action('--flag', 'flag', nargs=nargs), + OptionType(file=False, choice=False, unknown_required=True), + ) + for nargs in ('+', '*') + ), + *( + ( + Action('--flag', 'flag', metavar=metavar), + OptionType(file=False, choice=False, unknown_required=True), + ) + for metavar in ('PATTERN', 'KEYS', 'N') + ), + *( + ( + Action('--flag', 'flag', type=type, default=None), + OptionType(file=False, choice=False, unknown_required=True), + ) + for type in (int, str) + ), + ( + Action('--flag', 'flag', type=int, default=1), + OptionType(file=False, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', type=str, required=True, metavar='PATH'), + OptionType(file=True, choice=False, unknown_required=True), + ), + ( + Action('--flag', 'flag', type=str, required=True, metavar='PATH', default='/dev/null'), + OptionType(file=True, choice=False, unknown_required=True), + ), + ( + Action('--flag', 'flag', type=str, required=False, metavar='PATH', default='/dev/null'), + OptionType(file=True, choice=False, unknown_required=False), + ), ] -@pytest.mark.parametrize('action, expected', file_options_test_data) -def test_has_file_options_detects_file_options(action: Action, expected: bool): - assert has_file_options(action) == expected +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_file_options_detects_file_options(action: Action, option_type: OptionType): + assert has_file_options(action) == option_type.file # if has_file_options(action) was true, has_exact_options(action) should also be true - if expected: + if option_type.file: assert has_exact_options(action) -choices_test_data = [ - (Action('--flag', 'flag'), False), - (Action('--flag', 'flag', choices=['a', 'b']), True), - (Action('--flag', 'flag', choices=None), False), -] - - -@pytest.mark.parametrize('action, expected', choices_test_data) -def test_has_choice_options_detects_choice_options(action: Action, expected: bool): - assert has_choice_options(action) == expected +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_choice_options_detects_choice_options(action: Action, option_type: OptionType): + assert has_choice_options(action) == option_type.choice # if has_choice_options(action) was true, has_exact_options(action) should also be true - if expected: + if option_type.choice: assert has_exact_options(action) -unknown_required_param_test_data = [ - (Action('--flag', 'flag'), False), - (Action('--flag', 'flag', required=True), True), - *((Action('--flag', 'flag', nargs=nargs), True) for nargs in ('+', '*')), - *((Action('--flag', 'flag', metavar=metavar), True) for metavar in ('PATTERN', 'KEYS', 'N')), - *((Action('--flag', 'flag', type=type, default=None), True) for type in (int, str)), - (Action('--flag', 'flag', type=int, default=1), False), -] - - -@pytest.mark.parametrize('action, expected', unknown_required_param_test_data) +@pytest.mark.parametrize('action, option_type', test_data) def test_has_unknown_required_param_options_detects_unknown_required_param_options( - action: Action, expected: bool + action: Action, option_type: OptionType ): - assert has_unknown_required_param_options(action) == expected + assert has_unknown_required_param_options(action) == option_type.unknown_required # if has_unknown_required_param_options(action) was true, has_exact_options(action) should also be true - if expected: + if option_type.unknown_required: assert has_exact_options(action) + + +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): + assert has_exact_options(action) == ( + option_type.file or option_type.choice or option_type.unknown_required + ) From aa564ac5fef2be1c13d1178c9b482f326c053baf Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:25:15 -0700 Subject: [PATCH 169/344] fix the error thrown, unit test for it, and add string explanations --- borgmatic/commands/completion.py | 2 +- tests/unit/commands/test_completions.py | 35 ++++++++++++++++--------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 702a09dd..65ce415b 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -136,7 +136,7 @@ def exact_options_completion(action: Action): if has_unknown_required_param_options(action): return f'''\ncomplete -c borgmatic -x -n "__borgmatic_last_arg {args}"''' - raise RuntimeError( + raise ValueError( f'Unexpected action: {action} passes has_exact_options but has no choices produced' ) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 51ab29ac..73623096 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -5,6 +5,7 @@ from typing import Tuple import pytest from borgmatic.commands.completion import ( + exact_options_completion, has_choice_options, has_exact_options, has_file_options, @@ -89,32 +90,40 @@ test_data: list[TestCase] = [ @pytest.mark.parametrize('action, option_type', test_data) def test_has_file_options_detects_file_options(action: Action, option_type: OptionType): - assert has_file_options(action) == option_type.file - # if has_file_options(action) was true, has_exact_options(action) should also be true - if option_type.file: - assert has_exact_options(action) + assert ( + has_file_options(action) == option_type.file + ), f'Action: {action} should be file={option_type.file}' @pytest.mark.parametrize('action, option_type', test_data) def test_has_choice_options_detects_choice_options(action: Action, option_type: OptionType): - assert has_choice_options(action) == option_type.choice - # if has_choice_options(action) was true, has_exact_options(action) should also be true - if option_type.choice: - assert has_exact_options(action) + assert ( + has_choice_options(action) == option_type.choice + ), f'Action: {action} should be choice={option_type.choice}' @pytest.mark.parametrize('action, option_type', test_data) def test_has_unknown_required_param_options_detects_unknown_required_param_options( action: Action, option_type: OptionType ): - assert has_unknown_required_param_options(action) == option_type.unknown_required - # if has_unknown_required_param_options(action) was true, has_exact_options(action) should also be true - if option_type.unknown_required: - assert has_exact_options(action) + assert ( + has_unknown_required_param_options(action) == option_type.unknown_required + ), f'Action: {action} should be unknown_required={option_type.unknown_required}' @pytest.mark.parametrize('action, option_type', test_data) def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): assert has_exact_options(action) == ( option_type.file or option_type.choice or option_type.unknown_required - ) + ), f'Action: {action} should have exact options given {option_type}' + + +@pytest.mark.parametrize('action, option_type', test_data) +def test_produce_exact_options_completion(action: Action, option_type: OptionType): + try: + completion = exact_options_completion(action) + assert ( + type(completion) == str + ), f'Completion should be a string, got {completion} of type {type(completion)}' + except ValueError as value_error: + assert False, f'exact_options_completion raised ValueError: {value_error}' From ccfdd6806f51fed91c6778eb3f66ab6313f4c7d8 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:29:14 -0700 Subject: [PATCH 170/344] test the value of completions --- tests/unit/commands/test_completions.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 73623096..74502775 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -114,7 +114,7 @@ def test_has_unknown_required_param_options_detects_unknown_required_param_optio @pytest.mark.parametrize('action, option_type', test_data) def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): assert has_exact_options(action) == ( - option_type.file or option_type.choice or option_type.unknown_required + True in option_type ), f'Action: {action} should have exact options given {option_type}' @@ -122,8 +122,12 @@ def test_has_exact_options_detects_exact_options(action: Action, option_type: Op def test_produce_exact_options_completion(action: Action, option_type: OptionType): try: completion = exact_options_completion(action) - assert ( - type(completion) == str - ), f'Completion should be a string, got {completion} of type {type(completion)}' + if True in option_type: + assert completion.startswith( + '\ncomplete -c borgmatic' + ), f'Completion should start with "complete -c borgmatic", got {completion}' + else: + assert completion == '', f'Completion should be empty, got {completion}' + except ValueError as value_error: assert False, f'exact_options_completion raised ValueError: {value_error}' From d7320599795bac4f6a2605fa7f630a5282e6f50d Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:32:10 -0700 Subject: [PATCH 171/344] fix rotted comments --- borgmatic/commands/completion.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 65ce415b..e39f742d 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -116,8 +116,9 @@ def has_exact_options(action: Action): def exact_options_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation that forces file completion or options - completion, if the action takes such an argument and was the last action on the command line. + Given an argparse.Action instance, return a completion invocation that forces file completions, options completion, + or just that some value follow the action, if the action takes such an argument and was the last action on the + command line prior to the cursor. Otherwise, return an empty string. ''' @@ -188,7 +189,7 @@ def fish_completion(): end __borgmatic_check_version - function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line' + function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line before the cursor' set -l all_args (commandline -poc) # premature optimization to avoid iterating all args if there aren't enough # to have a last arg beyond borgmatic From a047f856a13c7cce4f7463ad2d9fa0a01c695d76 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:37:38 -0700 Subject: [PATCH 172/344] tweak docstring, add comment --- borgmatic/commands/completion.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index e39f742d..555ffcf9 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -91,8 +91,8 @@ def has_unknown_required_param_options(action: Action): ''' A catch-all for options that take a required parameter, but we don't know what the parameter is. This should be used last. These are actions that take something like a glob, a list of numbers, or a string. - There is no way to know what the valid options are, but we need to prevent another argument from being shown, - and let the user know that they need to provide a parameter. + + Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid. ''' return ( action.required is True @@ -215,6 +215,7 @@ def fish_completion(): ) + ('\n# global flags',) + tuple( + # -n is checked in order, so put faster / more likely to be true checks first f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' for action in top_level_parser._actions if len(action.option_strings) > 0 From c8f4344f8968aa9c61a8c7f047b091f6f9854868 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:39:02 -0700 Subject: [PATCH 173/344] add more justification to checks --- borgmatic/commands/completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 555ffcf9..affe5f97 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -218,6 +218,7 @@ def fish_completion(): # -n is checked in order, so put faster / more likely to be true checks first f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' for action in top_level_parser._actions + # ignore the noargs action, as this is an impossible completion for fish if len(action.option_strings) > 0 if 'Deprecated' not in action.help ) From efb81fc2c1ea61fdabf77ecbba27214a359a0ec9 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:42:32 -0700 Subject: [PATCH 174/344] rename last arg helper function to current arg for clarity --- borgmatic/commands/completion.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index affe5f97..d0ce7452 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -129,13 +129,13 @@ def exact_options_completion(action: Action): args = ' '.join(action.option_strings) if has_file_options(action): - return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_last_arg {args}"''' + return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_current_arg {args}"''' if has_choice_options(action): - return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_last_arg {args}"''' + return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_current_arg {args}"''' if has_unknown_required_param_options(action): - return f'''\ncomplete -c borgmatic -x -n "__borgmatic_last_arg {args}"''' + return f'''\ncomplete -c borgmatic -x -n "__borgmatic_current_arg {args}"''' raise ValueError( f'Unexpected action: {action} passes has_exact_options but has no choices produced' @@ -189,7 +189,7 @@ def fish_completion(): end __borgmatic_check_version - function __borgmatic_last_arg --description 'Check if any of the given arguments are the last on the command line before the cursor' + function __borgmatic_current_arg --description 'Check if any of the given arguments are the last on the command line before the cursor' set -l all_args (commandline -poc) # premature optimization to avoid iterating all args if there aren't enough # to have a last arg beyond borgmatic @@ -205,7 +205,7 @@ def fish_completion(): end set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}" - set --local exact_option_condition "not __borgmatic_last_arg {' '.join(exact_option_args)}" + set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}" ''' ) + ('\n# subparser completions',) From 43c532bc577eb7169c5700fb42d74b805025ae66 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 11:51:35 -0700 Subject: [PATCH 175/344] add test for dedent strip --- borgmatic/commands/completion.py | 4 ++++ tests/unit/commands/test_completions.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index d0ce7452..4b2f17f3 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -143,6 +143,10 @@ def exact_options_completion(action: Action): def dedent_strip_as_tuple(string: str): + ''' + Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string. + Makes it easier to write multiline strings for completions when you join them with a tuple. + ''' return (dedent(string).strip('\n'),) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 74502775..69110af6 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -5,6 +5,7 @@ from typing import Tuple import pytest from borgmatic.commands.completion import ( + dedent_strip_as_tuple, exact_options_completion, has_choice_options, has_exact_options, @@ -131,3 +132,12 @@ def test_produce_exact_options_completion(action: Action, option_type: OptionTyp except ValueError as value_error: assert False, f'exact_options_completion raised ValueError: {value_error}' + + +def test_dedent_strip_as_tuple(): + dedent_strip_as_tuple( + ''' + a + b + ''' + ) From 0657106893b729a35227a8be7198b90982b214d9 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:46:15 -0700 Subject: [PATCH 176/344] clarify dedent test name --- tests/unit/commands/test_completions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 69110af6..b6d3103f 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -134,7 +134,7 @@ def test_produce_exact_options_completion(action: Action, option_type: OptionTyp assert False, f'exact_options_completion raised ValueError: {value_error}' -def test_dedent_strip_as_tuple(): +def test_dedent_strip_as_tuple_does_not_raise(): dedent_strip_as_tuple( ''' a From 453b78c852fb36186dc3a6cc0a3f00681e4757ee Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:49:07 -0700 Subject: [PATCH 177/344] drop messages --- tests/unit/commands/test_completions.py | 32 +++++++------------------ 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index b6d3103f..ba562209 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -91,47 +91,33 @@ test_data: list[TestCase] = [ @pytest.mark.parametrize('action, option_type', test_data) def test_has_file_options_detects_file_options(action: Action, option_type: OptionType): - assert ( - has_file_options(action) == option_type.file - ), f'Action: {action} should be file={option_type.file}' + assert has_file_options(action) == option_type.file @pytest.mark.parametrize('action, option_type', test_data) def test_has_choice_options_detects_choice_options(action: Action, option_type: OptionType): - assert ( - has_choice_options(action) == option_type.choice - ), f'Action: {action} should be choice={option_type.choice}' + assert has_choice_options(action) == option_type.choice @pytest.mark.parametrize('action, option_type', test_data) def test_has_unknown_required_param_options_detects_unknown_required_param_options( action: Action, option_type: OptionType ): - assert ( - has_unknown_required_param_options(action) == option_type.unknown_required - ), f'Action: {action} should be unknown_required={option_type.unknown_required}' + assert has_unknown_required_param_options(action) == option_type.unknown_required @pytest.mark.parametrize('action, option_type', test_data) def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): - assert has_exact_options(action) == ( - True in option_type - ), f'Action: {action} should have exact options given {option_type}' + assert has_exact_options(action) == (True in option_type) @pytest.mark.parametrize('action, option_type', test_data) def test_produce_exact_options_completion(action: Action, option_type: OptionType): - try: - completion = exact_options_completion(action) - if True in option_type: - assert completion.startswith( - '\ncomplete -c borgmatic' - ), f'Completion should start with "complete -c borgmatic", got {completion}' - else: - assert completion == '', f'Completion should be empty, got {completion}' - - except ValueError as value_error: - assert False, f'exact_options_completion raised ValueError: {value_error}' + completion = exact_options_completion(action) + if True in option_type: + assert completion.startswith('\ncomplete -c borgmatic') + else: + assert completion == '' def test_dedent_strip_as_tuple_does_not_raise(): From aa770b98f90652a3579cf4b21ae1a5b370a3c4b2 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:50:37 -0700 Subject: [PATCH 178/344] follow unit test module convention --- tests/unit/commands/test_completions.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index ba562209..64ea3c7e 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -4,14 +4,7 @@ from typing import Tuple import pytest -from borgmatic.commands.completion import ( - dedent_strip_as_tuple, - exact_options_completion, - has_choice_options, - has_exact_options, - has_file_options, - has_unknown_required_param_options, -) +from borgmatic.commands import completion as module OptionType = namedtuple('OptionType', ['file', 'choice', 'unknown_required']) TestCase = Tuple[Action, OptionType] @@ -91,29 +84,29 @@ test_data: list[TestCase] = [ @pytest.mark.parametrize('action, option_type', test_data) def test_has_file_options_detects_file_options(action: Action, option_type: OptionType): - assert has_file_options(action) == option_type.file + assert module.has_file_options(action) == option_type.file @pytest.mark.parametrize('action, option_type', test_data) def test_has_choice_options_detects_choice_options(action: Action, option_type: OptionType): - assert has_choice_options(action) == option_type.choice + assert module.has_choice_options(action) == option_type.choice @pytest.mark.parametrize('action, option_type', test_data) def test_has_unknown_required_param_options_detects_unknown_required_param_options( action: Action, option_type: OptionType ): - assert has_unknown_required_param_options(action) == option_type.unknown_required + assert module.has_unknown_required_param_options(action) == option_type.unknown_required @pytest.mark.parametrize('action, option_type', test_data) def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): - assert has_exact_options(action) == (True in option_type) + assert module.has_exact_options(action) == (True in option_type) @pytest.mark.parametrize('action, option_type', test_data) def test_produce_exact_options_completion(action: Action, option_type: OptionType): - completion = exact_options_completion(action) + completion = module.exact_options_completion(action) if True in option_type: assert completion.startswith('\ncomplete -c borgmatic') else: @@ -121,7 +114,7 @@ def test_produce_exact_options_completion(action: Action, option_type: OptionTyp def test_dedent_strip_as_tuple_does_not_raise(): - dedent_strip_as_tuple( + module.dedent_strip_as_tuple( ''' a b From 614c1bf2e41ee189188a5a8c9ec85079a2d98989 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:52:42 -0700 Subject: [PATCH 179/344] rename test to make function under test clearer --- tests/unit/commands/test_completions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 64ea3c7e..878e8204 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -105,7 +105,7 @@ def test_has_exact_options_detects_exact_options(action: Action, option_type: Op @pytest.mark.parametrize('action, option_type', test_data) -def test_produce_exact_options_completion(action: Action, option_type: OptionType): +def test_exact_options_completion_produces_reasonable_completions(action: Action, option_type: OptionType): completion = module.exact_options_completion(action) if True in option_type: assert completion.startswith('\ncomplete -c borgmatic') From 66964f613c1c8cd6e7fd3dece33cde430c8ed6b6 Mon Sep 17 00:00:00 2001 From: Isaac Date: Sat, 6 May 2023 15:56:50 -0700 Subject: [PATCH 180/344] formatting! --- tests/unit/commands/test_completions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index 878e8204..acb01f6a 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -105,7 +105,9 @@ def test_has_exact_options_detects_exact_options(action: Action, option_type: Op @pytest.mark.parametrize('action, option_type', test_data) -def test_exact_options_completion_produces_reasonable_completions(action: Action, option_type: OptionType): +def test_exact_options_completion_produces_reasonable_completions( + action: Action, option_type: OptionType +): completion = module.exact_options_completion(action) if True in option_type: assert completion.startswith('\ncomplete -c borgmatic') From 1a956e8b05a4700fecd503bfcc2e86be3dc3cb7b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 6 May 2023 16:04:15 -0700 Subject: [PATCH 181/344] Add fish shell completions to NEWS (#686). --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 30a2b9b0..9971da7e 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,9 @@ * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. + * #686: Add fish shell completion script so you can tab-complete on the borgmatic command-line. See + the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install --editable" development installs. * Run "borgmatic borg" action without capturing output so interactive prompts and flags like From e84bac29e580d782c54980e86a6c90f93d8be318 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 6 May 2023 16:18:37 -0700 Subject: [PATCH 182/344] Remove value type for compatibility with Python 3.8 (#686). --- tests/unit/commands/test_completions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index acb01f6a..ec596988 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -9,7 +9,7 @@ from borgmatic.commands import completion as module OptionType = namedtuple('OptionType', ['file', 'choice', 'unknown_required']) TestCase = Tuple[Action, OptionType] -test_data: list[TestCase] = [ +test_data = [ (Action('--flag', 'flag'), OptionType(file=False, choice=False, unknown_required=False)), *( ( From 15ef37d89fe5866ec2f248014b29a25393976c7a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 6 May 2023 16:25:26 -0700 Subject: [PATCH 183/344] Add test coverage for exact_options_completion() raising (#686). --- tests/unit/commands/test_completions.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py index ec596988..12829d5f 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/test_completions.py @@ -3,6 +3,7 @@ from collections import namedtuple from typing import Tuple import pytest +from flexmock import flexmock from borgmatic.commands import completion as module @@ -115,6 +116,16 @@ def test_exact_options_completion_produces_reasonable_completions( assert completion == '' +def test_exact_options_completion_raises_for_unexpected_action(): + flexmock(module).should_receive('has_exact_options').and_return(True) + flexmock(module).should_receive('has_file_options').and_return(False) + flexmock(module).should_receive('has_choice_options').and_return(False) + flexmock(module).should_receive('has_unknown_required_param_options').and_return(False) + + with pytest.raises(ValueError): + module.exact_options_completion(Action('--unknown', dest='unknown')) + + def test_dedent_strip_as_tuple_does_not_raise(): module.dedent_strip_as_tuple( ''' From b3b08ee6d776afbb2e6f05d6623108c3c2305739 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 7 May 2023 21:21:35 -0700 Subject: [PATCH 184/344] Fix error in "borgmatic restore" action when the configured repository path is relative (#691). --- NEWS | 2 ++ borgmatic/borg/extract.py | 5 +++- tests/unit/borg/test_extract.py | 45 +++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 9971da7e..8be142b9 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,8 @@ https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install --editable" development installs. + * #691: Fix error in "borgmatic restore" action when the configured repository path is relative + instead of absolute. * Run "borgmatic borg" action without capturing output so interactive prompts and flags like "--progress" still work. diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index f9471416..1b10ba26 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -2,6 +2,7 @@ import logging import os import subprocess +import borgmatic.config.validate from borgmatic.borg import environment, feature, flags, rlist from borgmatic.execute import DO_NOT_CAPTURE, execute_command @@ -109,7 +110,9 @@ def extract_archive( + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( - repository, + # Make the repository path absolute so the working directory changes below don't + # prevent Borg from finding the repo. + borgmatic.config.validate.normalize_repository_path(repository), archive, local_borg_version, ) diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index 26fd7380..6517379e 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -121,6 +121,9 @@ def test_extract_archive_calls_borg_with_path_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -140,6 +143,9 @@ def test_extract_archive_calls_borg_with_remote_path_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -167,6 +173,9 @@ def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -186,6 +195,9 @@ def test_extract_archive_calls_borg_with_umask_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -205,6 +217,9 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -225,6 +240,9 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -247,6 +265,9 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -266,6 +287,9 @@ def test_extract_archive_calls_borg_with_dry_run_parameter(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=True, @@ -285,6 +309,9 @@ def test_extract_archive_calls_borg_with_destination_path(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -305,6 +332,9 @@ def test_extract_archive_calls_borg_with_strip_components(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -335,6 +365,9 @@ def test_extract_archive_calls_borg_with_strip_components_calculated_from_all(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -354,6 +387,9 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') flexmock(module).should_receive('execute_command').never() with pytest.raises(ValueError): @@ -382,6 +418,9 @@ def test_extract_archive_calls_borg_with_progress_parameter(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -427,6 +466,9 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') assert ( module.extract_archive( @@ -455,6 +497,9 @@ def test_extract_archive_skips_abspath_for_remote_repository(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('server:repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, From 92a2230a07e9e6e3a6024c3e86dbc3cfe7208187 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 8 May 2023 23:00:49 -0700 Subject: [PATCH 185/344] Add support for logging each log line as a JSON object via global "--log-json" flag (#680). --- NEWS | 1 + borgmatic/actions/borg.py | 2 + borgmatic/actions/break_lock.py | 2 + borgmatic/actions/check.py | 1 + borgmatic/actions/compact.py | 1 + borgmatic/actions/create.py | 1 + borgmatic/actions/export_tar.py | 2 + borgmatic/actions/extract.py | 2 + borgmatic/actions/info.py | 9 +- borgmatic/actions/list.py | 9 +- borgmatic/actions/mount.py | 3 + borgmatic/actions/prune.py | 1 + borgmatic/actions/rcreate.py | 1 + borgmatic/actions/restore.py | 11 +- borgmatic/actions/rinfo.py | 2 + borgmatic/actions/rlist.py | 2 + borgmatic/actions/transfer.py | 1 + borgmatic/borg/break_lock.py | 6 +- borgmatic/borg/check.py | 11 +- borgmatic/borg/compact.py | 2 + borgmatic/borg/create.py | 2 + borgmatic/borg/export_tar.py | 2 + borgmatic/borg/extract.py | 24 +++-- borgmatic/borg/info.py | 8 +- borgmatic/borg/list.py | 24 +++-- borgmatic/borg/mount.py | 6 +- borgmatic/borg/prune.py | 2 + borgmatic/borg/rcreate.py | 5 + borgmatic/borg/rinfo.py | 8 +- borgmatic/borg/rlist.py | 30 +++--- borgmatic/borg/transfer.py | 5 +- borgmatic/commands/arguments.py | 5 + borgmatic/commands/borgmatic.py | 11 +- tests/integration/borg/test_commands.py | 26 ++++- tests/unit/actions/test_borg.py | 1 + tests/unit/actions/test_break_lock.py | 1 + tests/unit/actions/test_info.py | 1 + tests/unit/actions/test_list.py | 1 + tests/unit/actions/test_mount.py | 1 + tests/unit/actions/test_restore.py | 3 + tests/unit/actions/test_rinfo.py | 1 + tests/unit/actions/test_rlist.py | 1 + tests/unit/borg/test_break_lock.py | 18 ++++ tests/unit/borg/test_check.py | 46 ++++++++ tests/unit/borg/test_compact.py | 45 +++++++- tests/unit/borg/test_create.py | 89 +++++++++++++++ tests/unit/borg/test_export_tar.py | 33 ++++++ tests/unit/borg/test_extract.py | 98 ++++++++++++++--- tests/unit/borg/test_info.py | 45 ++++++++ tests/unit/borg/test_list.py | 62 ++++++++++- tests/unit/borg/test_mount.py | 31 ++++++ tests/unit/borg/test_prune.py | 46 ++++++-- tests/unit/borg/test_rcreate.py | 57 ++++++++++ tests/unit/borg/test_rinfo.py | 49 +++++++-- tests/unit/borg/test_rlist.py | 138 +++++++++++++++++++++--- tests/unit/borg/test_transfer.py | 44 ++++++++ 56 files changed, 934 insertions(+), 105 deletions(-) diff --git a/NEWS b/NEWS index 8be142b9..17560d7e 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,7 @@ commands with arguments. * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break database dumping. + * #680: Add support for logging each log line as a JSON object via global "--log-json" flag. * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index 3d2998b6..ec445fbb 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -12,6 +12,7 @@ def run_borg( storage, local_borg_version, borg_arguments, + global_arguments, local_path, remote_path, ): @@ -27,6 +28,7 @@ def run_borg( borg_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index 2174161c..f049e772 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -11,6 +11,7 @@ def run_break_lock( storage, local_borg_version, break_lock_arguments, + global_arguments, local_path, remote_path, ): @@ -25,6 +26,7 @@ def run_break_lock( repository['path'], storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 1696e07d..aac536e3 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -44,6 +44,7 @@ def run_check( storage, consistency, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, progress=check_arguments.progress, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 95334c52..24b30c0e 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -45,6 +45,7 @@ def run_compact( repository['path'], storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, progress=compact_arguments.progress, diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 3fbe31e6..a3f8da57 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -67,6 +67,7 @@ def run_create( location, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, progress=create_arguments.progress, diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index ff9f31ba..798bd418 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -33,6 +33,7 @@ def run_export_tar( export_tar_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ), @@ -40,6 +41,7 @@ def run_export_tar( export_tar_arguments.destination, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index cc1516ce..1f4317cd 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -44,6 +44,7 @@ def run_extract( extract_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ), @@ -51,6 +52,7 @@ def run_extract( location, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, destination_path=extract_arguments.destination, diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index 54023127..d138dbd4 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -13,6 +13,7 @@ def run_info( storage, local_borg_version, info_arguments, + global_arguments, local_path, remote_path, ): @@ -31,6 +32,7 @@ def run_info( info_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -38,9 +40,10 @@ def run_info( repository['path'], storage, local_borg_version, - info_arguments=info_arguments, - local_path=local_path, - remote_path=remote_path, + info_arguments, + global_arguments, + local_path, + remote_path, ) if json_output: # pragma: nocover yield json.loads(json_output) diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 359f3b67..548f1979 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -12,6 +12,7 @@ def run_list( storage, local_borg_version, list_arguments, + global_arguments, local_path, remote_path, ): @@ -33,6 +34,7 @@ def run_list( list_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -40,9 +42,10 @@ def run_list( repository['path'], storage, local_borg_version, - list_arguments=list_arguments, - local_path=local_path, - remote_path=remote_path, + list_arguments, + global_arguments, + local_path, + remote_path, ) if json_output: # pragma: nocover yield json.loads(json_output) diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index 72e321a0..60f7f23c 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -12,6 +12,7 @@ def run_mount( storage, local_borg_version, mount_arguments, + global_arguments, local_path, remote_path, ): @@ -33,6 +34,7 @@ def run_mount( mount_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ), @@ -42,6 +44,7 @@ def run_mount( mount_arguments.options, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 9a5d936b..2e25264b 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -44,6 +44,7 @@ def run_prune( storage, retention, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, stats=prune_arguments.stats, diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index 62206318..a3015c61 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -29,6 +29,7 @@ def run_rcreate( repository['path'], storage, local_borg_version, + global_arguments, rcreate_arguments.encryption_mode, rcreate_arguments.source_repository, rcreate_arguments.copy_crypt_key, diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index f061dca8..246c11a6 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -93,6 +93,7 @@ def restore_single_database( location_config=location, storage_config=storage, local_borg_version=local_borg_version, + global_arguments=global_arguments, local_path=local_path, remote_path=remote_path, destination_path='/', @@ -119,14 +120,15 @@ def collect_archive_database_names( location, storage, local_borg_version, + global_arguments, local_path, remote_path, ): ''' Given a local or remote repository path, a resolved archive name, a location configuration dict, - a storage configuration dict, the local Borg version, and local and remote Borg paths, query the - archive for the names of databases it contains and return them as a dict from hook name to a - sequence of database names. + a storage configuration dict, the local Borg version, global_arguments an argparse.Namespace, + and local and remote Borg paths, query the archive for the names of databases it contains and + return them as a dict from hook name to a sequence of database names. ''' borgmatic_source_directory = os.path.expanduser( location.get( @@ -141,6 +143,7 @@ def collect_archive_database_names( archive, storage, local_borg_version, + global_arguments, list_path=parent_dump_path, local_path=local_path, remote_path=remote_path, @@ -279,6 +282,7 @@ def run_restore( restore_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -288,6 +292,7 @@ def run_restore( location, storage, local_borg_version, + global_arguments, local_path, remote_path, ) diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 0947ec3d..279cd0e7 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -12,6 +12,7 @@ def run_rinfo( storage, local_borg_version, rinfo_arguments, + global_arguments, local_path, remote_path, ): @@ -31,6 +32,7 @@ def run_rinfo( storage, local_borg_version, rinfo_arguments=rinfo_arguments, + global_arguments=global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index 10d06a51..50c59b6f 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -12,6 +12,7 @@ def run_rlist( storage, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ): @@ -31,6 +32,7 @@ def run_rlist( storage, local_borg_version, rlist_arguments=rlist_arguments, + global_arguments=global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 8089fd4e..36ac166d 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -24,6 +24,7 @@ def run_transfer( storage, local_borg_version, transfer_arguments, + global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 7099af83..3c361956 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -10,13 +10,14 @@ def break_lock( repository_path, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage configuration dict, the local Borg version, - and optional local and remote Borg paths, break any repository and cache locks leftover from Borg - aborting. + an argparse.Namespace of global arguments, and optional local and remote Borg paths, break any + repository and cache locks leftover from Borg aborting. ''' umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) @@ -25,6 +26,7 @@ def break_lock( (local_path, 'break-lock') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index cee9d923..52d5208c 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -259,6 +259,7 @@ def check_archives( storage_config, consistency_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, progress=None, @@ -283,6 +284,7 @@ def check_archives( storage_config, local_borg_version, argparse.Namespace(json=True), + global_arguments, local_path, remote_path, ) @@ -317,6 +319,7 @@ def check_archives( + (('--repair',) if repair else ()) + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix) + (('--remote-path', remote_path) if remote_path else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags + (('--progress',) if progress else ()) @@ -340,6 +343,12 @@ def check_archives( if 'extract' in checks: extract.extract_last_archive_dry_run( - storage_config, local_borg_version, repository_path, lock_wait, local_path, remote_path + storage_config, + local_borg_version, + global_arguments, + repository_path, + lock_wait, + local_path, + remote_path, ) write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract')) diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index 0e9d3e89..24f37ee3 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -11,6 +11,7 @@ def compact_segments( repository_path, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, progress=False, @@ -29,6 +30,7 @@ def compact_segments( (local_path, 'compact') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--progress',) if progress else ()) + (('--cleanup-commits',) if cleanup_commits else ()) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 7413dfc5..e3b70eb5 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -326,6 +326,7 @@ def create_archive( location_config, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, progress=False, @@ -438,6 +439,7 @@ def create_archive( + (('--files-cache', files_cache) if files_cache else ()) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--list', '--filter', list_filter_flags) diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index a624f07d..b6d9a04c 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -15,6 +15,7 @@ def export_tar_archive( destination_path, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, tar_filter=None, @@ -38,6 +39,7 @@ def export_tar_archive( (local_path, 'export-tar') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--list',) if list_files else ()) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 1b10ba26..d5465bb9 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -12,6 +12,7 @@ logger = logging.getLogger(__name__) def extract_last_archive_dry_run( storage_config, local_borg_version, + global_arguments, repository_path, lock_wait=None, local_path='borg', @@ -21,8 +22,6 @@ def extract_last_archive_dry_run( Perform an extraction dry-run of the most recent archive. If there are no archives, skip the dry-run. ''' - remote_path_flags = ('--remote-path', remote_path) if remote_path else () - lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else () verbosity_flags = () if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') @@ -31,7 +30,13 @@ def extract_last_archive_dry_run( try: last_archive_name = rlist.resolve_archive_name( - repository_path, 'latest', storage_config, local_borg_version, local_path, remote_path + repository_path, + 'latest', + storage_config, + local_borg_version, + global_arguments, + local_path, + remote_path, ) except ValueError: logger.warning('No archives found. Skipping extract consistency check.') @@ -41,8 +46,9 @@ def extract_last_archive_dry_run( borg_environment = environment.make_environment(storage_config) full_extract_command = ( (local_path, 'extract', '--dry-run') - + remote_path_flags - + lock_wait_flags + + (('--remote-path', remote_path) if remote_path else ()) + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags + list_flag + flags.make_repository_archive_flags( @@ -63,6 +69,7 @@ def extract_archive( location_config, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, destination_path=None, @@ -72,9 +79,9 @@ def extract_archive( ): ''' Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to - restore from the archive, the local Borg version string, location/storage configuration dicts, - optional local and remote Borg paths, and an optional destination path to extract to, extract - the archive into the current directory. + restore from the archive, the local Borg version string, an argparse.Namespace of global + arguments, location/storage configuration dicts, optional local and remote Borg paths, and an + optional destination path to extract to, extract the archive into the current directory. If extract to stdout is True, then start the extraction streaming to stdout, and return that extract process as an instance of subprocess.Popen. @@ -102,6 +109,7 @@ def extract_archive( + (('--remote-path', remote_path) if remote_path else ()) + numeric_ids_flags + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index ef2c0c44..91520e00 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -12,13 +12,14 @@ def display_archives_info( storage_config, local_borg_version, info_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, and the - arguments to the info action, display summary information for Borg archives in the repository or - return JSON summary information. + Given a local or remote repository path, a storage config dict, the local Borg version, global + arguments as an argparse.Namespace, and the arguments to the info action, display summary + information for Borg archives in the repository or return JSON summary information. ''' borgmatic.logger.add_custom_log_levels() lock_wait = storage_config.get('lock_wait', None) @@ -36,6 +37,7 @@ def display_archives_info( else () ) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', lock_wait) + ( ( diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 908f8fef..96a6a87f 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -25,6 +25,7 @@ def make_list_command( storage_config, local_borg_version, list_arguments, + global_arguments, local_path='borg', remote_path=None, ): @@ -48,6 +49,7 @@ def make_list_command( else () ) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', lock_wait) + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES) + ( @@ -90,14 +92,16 @@ def capture_archive_listing( archive, storage_config, local_borg_version, + global_arguments, list_path=None, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, an archive name, a storage config dict, the local Borg - version, the archive path in which to list files, and local and remote Borg paths, capture the - output of listing that archive and return it as a list of file paths. + version, global arguments as an argparse.Namespace, the archive path in which to list files, and + local and remote Borg paths, capture the output of listing that archive and return it as a list + of file paths. ''' borg_environment = environment.make_environment(storage_config) @@ -115,6 +119,7 @@ def capture_archive_listing( json=None, format='{path}{NL}', # noqa: FS003 ), + global_arguments, local_path, remote_path, ), @@ -130,15 +135,17 @@ def list_archive( storage_config, local_borg_version, list_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, the - arguments to the list action, and local and remote Borg paths, display the output of listing - the files of a Borg archive (or return JSON output). If list_arguments.find_paths are given, - list the files by searching across multiple archives. If neither find_paths nor archive name - are given, instead list the archives in the given repository. + Given a local or remote repository path, a storage config dict, the local Borg version, global + arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace, + and local and remote Borg paths, display the output of listing the files of a Borg archive (or + return JSON output). If list_arguments.find_paths are given, list the files by searching across + multiple archives. If neither find_paths nor archive name are given, instead list the archives + in the given repository. ''' borgmatic.logger.add_custom_log_levels() @@ -164,6 +171,7 @@ def list_archive( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ) @@ -205,6 +213,7 @@ def list_archive( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ), @@ -233,6 +242,7 @@ def list_archive( storage_config, local_borg_version, archive_arguments, + global_arguments, local_path, remote_path, ) + make_find_paths(list_arguments.find_paths) diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 07a6c632..6ce01a87 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -15,14 +15,15 @@ def mount_archive( options, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, an optional archive name, a filesystem mount point, zero or more paths to mount from the archive, extra Borg mount options, a storage configuration - dict, the local Borg version, and optional local and remote Borg paths, mount the archive onto - the mount point. + dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional + local and remote Borg paths, mount the archive onto the mount point. ''' umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) @@ -31,6 +32,7 @@ def mount_archive( (local_path, 'mount') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 3f06dc2d..b6be75ba 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -53,6 +53,7 @@ def prune_archives( storage_config, retention_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, stats=False, @@ -73,6 +74,7 @@ def prune_archives( + make_prune_flags(storage_config, retention_config, local_borg_version) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--stats',) if stats and not dry_run else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) diff --git a/borgmatic/borg/rcreate.py b/borgmatic/borg/rcreate.py index 7510529d..54a865c5 100644 --- a/borgmatic/borg/rcreate.py +++ b/borgmatic/borg/rcreate.py @@ -16,6 +16,7 @@ def create_repository( repository_path, storage_config, local_borg_version, + global_arguments, encryption_mode, source_repository=None, copy_crypt_key=False, @@ -37,6 +38,7 @@ def create_repository( storage_config, local_borg_version, argparse.Namespace(json=True), + global_arguments, local_path, remote_path, ) @@ -46,6 +48,7 @@ def create_repository( if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE: raise + lock_wait = storage_config.get('lock_wait') extra_borg_options = storage_config.get('extra_borg_options', {}).get('rcreate', '') rcreate_command = ( @@ -63,6 +66,8 @@ def create_repository( + (('--make-parent-dirs',) if make_parent_dirs else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--remote-path', remote_path) if remote_path else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index 97d7a666..e1542d28 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -12,13 +12,14 @@ def display_repository_info( storage_config, local_borg_version, rinfo_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, and the - arguments to the rinfo action, display summary information for the Borg repository or return - JSON summary information. + Given a local or remote repository path, a storage config dict, the local Borg version, the + arguments to the rinfo action, and global arguments as an argparse.Namespace, display summary + information for the Borg repository or return JSON summary information. ''' borgmatic.logger.add_custom_log_levels() lock_wait = storage_config.get('lock_wait', None) @@ -41,6 +42,7 @@ def display_repository_info( else () ) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', lock_wait) + (('--json',) if rinfo_arguments.json else ()) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 7f468705..ba45aa0a 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -12,28 +12,29 @@ def resolve_archive_name( archive, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, an archive name, a storage config dict, a local Borg - path, and a remote Borg path, return the archive name. But if the archive name is "latest", - then instead introspect the repository for the latest archive and return its name. + Given a local or remote repository path, an archive name, a storage config dict, the local Borg + version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path, + return the archive name. But if the archive name is "latest", then instead introspect the + repository for the latest archive and return its name. Raise ValueError if "latest" is given but there are no archives in the repository. ''' if archive != 'latest': return archive - lock_wait = storage_config.get('lock_wait', None) - full_command = ( ( local_path, 'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list', ) + flags.make_flags('remote-path', remote_path) - + flags.make_flags('lock-wait', lock_wait) + + flags.make_flags('log-json', global_arguments.log_json) + + flags.make_flags('lock-wait', storage_config.get('lock_wait')) + flags.make_flags('last', 1) + ('--short',) + flags.make_repository_flags(repository_path, local_borg_version) @@ -61,16 +62,15 @@ def make_rlist_command( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage config dict, the local Borg version, the - arguments to the rlist action, and local and remote Borg paths, return a command as a tuple to - list archives with a repository. + arguments to the rlist action, global arguments as an argparse.Namespace instance, and local and + remote Borg paths, return a command as a tuple to list archives with a repository. ''' - lock_wait = storage_config.get('lock_wait', None) - return ( ( local_path, @@ -87,7 +87,8 @@ def make_rlist_command( else () ) + flags.make_flags('remote-path', remote_path) - + flags.make_flags('lock-wait', lock_wait) + + flags.make_flags('log-json', global_arguments.log_json) + + flags.make_flags('lock-wait', storage_config.get('lock_wait')) + ( ( flags.make_flags('match-archives', f'sh:{rlist_arguments.prefix}*') @@ -113,13 +114,15 @@ def list_repository( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage config dict, the local Borg version, the - arguments to the list action, and local and remote Borg paths, display the output of listing - Borg archives in the given repository (or return JSON output). + arguments to the list action, global arguments as an argparse.Namespace instance, and local and + remote Borg paths, display the output of listing Borg archives in the given repository (or + return JSON output). ''' borgmatic.logger.add_custom_log_levels() borg_environment = environment.make_environment(storage_config) @@ -129,6 +132,7 @@ def list_repository( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 9fd05b76..d8f3978f 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -13,12 +13,14 @@ def transfer_archives( storage_config, local_borg_version, transfer_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg - version, and the arguments to the transfer action, transfer archives to the given repository. + version, the arguments to the transfer action, and global arguments as an argparse.Namespace + instance, transfer archives to the given repository. ''' borgmatic.logger.add_custom_log_levels() @@ -27,6 +29,7 @@ def transfer_archives( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', storage_config.get('lock_wait', None)) + ( flags.make_flags_from_arguments( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e5b99ec5..0812edea 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -187,6 +187,11 @@ def make_parsers(): type=str, help='Log format string used for log messages written to the log file', ) + global_group.add_argument( + '--log-json', + action='store_true', + help='Write log messages and console output as one JSON object per log line instead of formatted text', + ) global_group.add_argument( '--override', metavar='SECTION.OPTION=VALUE', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index d5fedba7..44396cd4 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -398,7 +398,8 @@ def run_actions( repository, storage, local_borg_version, - arguments['mount'], + action_arguments, + global_arguments, local_path, remote_path, ) @@ -420,6 +421,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -429,6 +431,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -438,6 +441,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -447,6 +451,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -455,7 +460,8 @@ def run_actions( repository, storage, local_borg_version, - arguments['break-lock'], + action_arguments, + global_arguments, local_path, remote_path, ) @@ -465,6 +471,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 1afb0e0f..44403193 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -1,3 +1,4 @@ +import argparse import copy from flexmock import flexmock @@ -58,7 +59,12 @@ def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): continue borgmatic.borg.transfer.transfer_archives( - False, 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + False, + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + global_arguments=flexmock(log_json=False), ) @@ -70,7 +76,11 @@ def test_make_list_command_does_not_duplicate_flags_or_raise(): continue command = borgmatic.borg.list.make_list_command( - 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + argparse.Namespace(log_json=False), ) assert_command_does_not_duplicate_flags(command) @@ -84,7 +94,11 @@ def test_make_rlist_command_does_not_duplicate_flags_or_raise(): continue command = borgmatic.borg.rlist.make_rlist_command( - 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + global_arguments=flexmock(log_json=True), ) assert_command_does_not_duplicate_flags(command) @@ -104,5 +118,9 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): continue borgmatic.borg.info.display_archives_info( - 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + argparse.Namespace(log_json=False), ) diff --git a/tests/unit/actions/test_borg.py b/tests/unit/actions/test_borg.py index f597acbf..2e03ec9c 100644 --- a/tests/unit/actions/test_borg.py +++ b/tests/unit/actions/test_borg.py @@ -16,6 +16,7 @@ def test_run_borg_does_not_raise(): repository={'path': 'repos'}, storage={}, local_borg_version=None, + global_arguments=flexmock(log_json=False), borg_arguments=borg_arguments, local_path=None, remote_path=None, diff --git a/tests/unit/actions/test_break_lock.py b/tests/unit/actions/test_break_lock.py index 6dc2470e..5949d7c1 100644 --- a/tests/unit/actions/test_break_lock.py +++ b/tests/unit/actions/test_break_lock.py @@ -14,6 +14,7 @@ def test_run_break_lock_does_not_raise(): storage={}, local_borg_version=None, break_lock_arguments=break_lock_arguments, + global_arguments=flexmock(), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py index a4f1d544..97161968 100644 --- a/tests/unit/actions/test_info.py +++ b/tests/unit/actions/test_info.py @@ -18,6 +18,7 @@ def test_run_info_does_not_raise(): storage={}, local_borg_version=None, info_arguments=info_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py index bfdfd010..5ee72251 100644 --- a/tests/unit/actions/test_list.py +++ b/tests/unit/actions/test_list.py @@ -18,6 +18,7 @@ def test_run_list_does_not_raise(): storage={}, local_borg_version=None, list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_mount.py b/tests/unit/actions/test_mount.py index 7eadfca1..743747d2 100644 --- a/tests/unit/actions/test_mount.py +++ b/tests/unit/actions/test_mount.py @@ -21,6 +21,7 @@ def test_run_mount_does_not_raise(): storage={}, local_borg_version=None, mount_arguments=mount_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 16fe2920..4bad6f82 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -72,6 +72,7 @@ def test_collect_archive_database_names_parses_archive_paths(): location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), local_path=flexmock(), remote_path=flexmock(), ) @@ -97,6 +98,7 @@ def test_collect_archive_database_names_parses_directory_format_archive_paths(): location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), local_path=flexmock(), remote_path=flexmock(), ) @@ -118,6 +120,7 @@ def test_collect_archive_database_names_skips_bad_archive_paths(): location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), local_path=flexmock(), remote_path=flexmock(), ) diff --git a/tests/unit/actions/test_rinfo.py b/tests/unit/actions/test_rinfo.py index 133e61ac..7b2371a3 100644 --- a/tests/unit/actions/test_rinfo.py +++ b/tests/unit/actions/test_rinfo.py @@ -15,6 +15,7 @@ def test_run_rinfo_does_not_raise(): storage={}, local_borg_version=None, rinfo_arguments=rinfo_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_rlist.py b/tests/unit/actions/test_rlist.py index 7f8b58aa..4a59dc30 100644 --- a/tests/unit/actions/test_rlist.py +++ b/tests/unit/actions/test_rlist.py @@ -15,6 +15,7 @@ def test_run_rlist_does_not_raise(): storage={}, local_borg_version=None, rlist_arguments=rlist_arguments, + global_arguments=flexmock(), local_path=None, remote_path=None, ) diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 509fc1b8..3dc55672 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -24,6 +24,7 @@ def test_break_lock_calls_borg_with_required_flags(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -35,6 +36,7 @@ def test_break_lock_calls_borg_with_remote_path_flags(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -47,6 +49,19 @@ def test_break_lock_calls_borg_with_umask_flags(): repository_path='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_break_lock_calls_borg_with_log_json_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(('borg', 'break-lock', '--log-json', 'repo')) + + module.break_lock( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -58,6 +73,7 @@ def test_break_lock_calls_borg_with_lock_wait_flags(): repository_path='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -70,6 +86,7 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -82,4 +99,5 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 1f992d3d..4cd6aa77 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -421,6 +421,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -451,6 +452,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), repair=True, ) @@ -490,6 +492,7 @@ def test_check_archives_calls_borg_with_parameters(checks): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -510,6 +513,7 @@ def test_check_archives_with_json_error_raises(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -528,6 +532,7 @@ def test_check_archives_with_missing_json_keys_raises(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -552,6 +557,7 @@ def test_check_archives_with_extract_check_calls_extract_only(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -576,6 +582,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -600,6 +607,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -618,6 +626,7 @@ def test_check_archives_without_any_checks_bails(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -648,6 +657,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -679,10 +689,43 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) +def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): + checks = ('repository',) + check_last = flexmock() + storage_config = {} + consistency_config = {'check_last': check_last} + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module.rinfo).should_receive('display_repository_info').and_return( + '{"repository": {"id": "repo"}}' + ) + flexmock(module).should_receive('make_check_flags').with_args( + '1.2.3', + storage_config, + checks, + check_last, + None, + ).and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(('borg', 'check', '--log-json', 'repo')) + flexmock(module).should_receive('make_check_time_path') + flexmock(module).should_receive('write_check_time') + + module.check_archives( + repository_path='repo', + location_config={}, + storage_config=storage_config, + consistency_config=consistency_config, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + + def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): checks = ('repository',) check_last = flexmock() @@ -711,6 +754,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): storage_config=storage_config, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -738,6 +782,7 @@ def test_check_archives_with_retention_prefix(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -761,4 +806,5 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): storage_config={'extra_borg_options': {'check': '--extra --options'}}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 60447db6..beacf547 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -25,7 +25,11 @@ def test_compact_segments_calls_borg_with_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) module.compact_segments( - dry_run=False, repository_path='repo', storage_config={}, local_borg_version='1.2.3' + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -35,7 +39,11 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.compact_segments( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + dry_run=False, ) @@ -45,7 +53,11 @@ def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.compact_segments( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + dry_run=False, ) @@ -53,7 +65,11 @@ def test_compact_segments_with_dry_run_skips_borg_call(): flexmock(module).should_receive('execute_command').never() module.compact_segments( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=True + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + dry_run=True, ) @@ -66,6 +82,7 @@ def test_compact_segments_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -79,6 +96,7 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -92,6 +110,7 @@ def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -105,6 +124,7 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), cleanup_commits=True, ) @@ -118,6 +138,7 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), threshold=20, ) @@ -132,6 +153,20 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(COMPACT_COMMAND + ('--log-json', 'repo'), logging.INFO) + + module.compact_segments( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -145,6 +180,7 @@ def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -157,4 +193,5 @@ def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options( repository_path='repo', storage_config={'extra_borg_options': {'compact': '--extra --options'}}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 3728a0bf..e0462e2d 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -492,6 +492,7 @@ def test_create_archive_calls_borg_with_parameters(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -535,6 +536,7 @@ def test_create_archive_calls_borg_with_environment(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -580,6 +582,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -625,6 +628,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -668,6 +672,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -708,6 +713,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, ) @@ -752,6 +758,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -792,6 +799,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, ) @@ -835,6 +843,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -880,6 +889,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stats=True, ) @@ -923,6 +933,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte }, storage_config={'checkpoint_interval': 600}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -965,6 +976,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume }, storage_config={'checkpoint_volume': 1024}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1007,6 +1019,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param }, storage_config={'chunker_params': '1,2,3,4'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1049,6 +1062,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( }, storage_config={'compression': 'rle'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1097,6 +1111,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ }, storage_config={'upload_rate_limit': 100}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1142,6 +1157,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1185,6 +1201,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1234,6 +1251,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1287,6 +1305,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1342,6 +1361,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1396,6 +1416,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1450,6 +1471,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1493,6 +1515,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1535,6 +1558,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -1578,6 +1602,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -1621,6 +1646,50 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): }, storage_config={'umask': 740}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_create_archive_with_log_json_calls_borg_with_log_json_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('map_directories_to_devices').and_return({}) + flexmock(module).should_receive('expand_directories').and_return(()) + flexmock(module).should_receive('pattern_root_directories').and_return([]) + flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) + flexmock(module).should_receive('expand_home_directories').and_return(()) + flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module).should_receive('ensure_files_readable') + flexmock(module).should_receive('make_pattern_flags').and_return(()) + flexmock(module).should_receive('make_exclude_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'create', '--log-json') + REPO_ARCHIVE_WITH_PATHS, + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=None, + ) + + module.create_archive( + dry_run=False, + repository_path='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -1663,6 +1732,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): }, storage_config={'lock_wait': 5}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1705,6 +1775,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stats=True, ) @@ -1748,6 +1819,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), list_files=True, ) @@ -1797,6 +1869,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -1840,6 +1913,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -1900,6 +1974,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, stream_processes=processes, ) @@ -1964,6 +2039,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2031,6 +2107,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2095,6 +2172,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2135,6 +2213,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, ) @@ -2177,6 +2256,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, stats=True, ) @@ -2224,6 +2304,7 @@ def test_create_archive_with_source_directories_glob_expands(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2267,6 +2348,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2309,6 +2391,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2351,6 +2434,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): }, storage_config={'archive_name_format': 'ARCHIVE_NAME'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2394,6 +2478,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2437,6 +2522,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2479,6 +2565,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): }, storage_config={'extra_borg_options': {'create': '--extra --options'}}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2539,6 +2626,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2564,6 +2652,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index 92776dd4..5fb7bff2 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -38,6 +38,7 @@ def test_export_tar_archive_calls_borg_with_path_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -59,6 +60,7 @@ def test_export_tar_archive_calls_borg_with_local_path_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -81,6 +83,7 @@ def test_export_tar_archive_calls_borg_with_remote_path_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -103,6 +106,27 @@ def test_export_tar_archive_calls_borg_with_umask_parameters(): destination_path='test.tar', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_export_tar_archive_calls_borg_with_log_json_parameter(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + insert_execute_command_mock(('borg', 'export-tar', '--log-json', 'repo::archive', 'test.tar')) + + module.export_tar_archive( + dry_run=False, + repository_path='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -124,6 +148,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_parameters(): destination_path='test.tar', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -144,6 +169,7 @@ def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -166,6 +192,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -185,6 +212,7 @@ def test_export_tar_archive_calls_borg_with_dry_run_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -206,6 +234,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), tar_filter='bzip2', ) @@ -229,6 +258,7 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), list_files=True, ) @@ -251,6 +281,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components=5, ) @@ -271,6 +302,7 @@ def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -290,4 +322,5 @@ def test_export_tar_archive_calls_borg_with_stdout_destination_path(): destination_path='-', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index 6517379e..a4032f6c 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -25,7 +25,11 @@ def test_extract_last_archive_dry_run_calls_borg_with_last_archive(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -34,7 +38,11 @@ def test_extract_last_archive_dry_run_without_any_archives_should_not_raise(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(('repo',)) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -47,7 +55,11 @@ def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_paramet ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -62,7 +74,11 @@ def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_param ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -76,13 +92,14 @@ def test_extract_last_archive_dry_run_calls_borg_via_local_path(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), repository_path='repo', lock_wait=None, local_path='borg1', ) -def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): +def test_extract_last_archive_dry_run_calls_borg_with_remote_path_flags(): flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') insert_execute_command_mock( ('borg', 'extract', '--dry-run', '--remote-path', 'borg1', 'repo::archive') @@ -94,13 +111,30 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), repository_path='repo', lock_wait=None, remote_path='borg1', ) -def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): +def test_extract_last_archive_dry_run_calls_borg_with_log_json_flag(): + flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') + insert_execute_command_mock(('borg', 'extract', '--dry-run', '--log-json', 'repo::archive')) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + + module.extract_last_archive_dry_run( + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + repository_path='repo', + lock_wait=None, + ) + + +def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_flags(): flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') insert_execute_command_mock( ('borg', 'extract', '--dry-run', '--lock-wait', '5', 'repo::archive') @@ -110,11 +144,15 @@ def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=5 + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=5, ) -def test_extract_archive_calls_borg_with_path_parameters(): +def test_extract_archive_calls_borg_with_path_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', 'repo::archive', 'path1', 'path2')) flexmock(module.feature).should_receive('available').and_return(True) @@ -133,10 +171,11 @@ def test_extract_archive_calls_borg_with_path_parameters(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_calls_borg_with_remote_path_parameters(): +def test_extract_archive_calls_borg_with_remote_path_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', '--remote-path', 'borg1', 'repo::archive')) flexmock(module.feature).should_receive('available').and_return(True) @@ -155,6 +194,7 @@ def test_extract_archive_calls_borg_with_remote_path_parameters(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -185,10 +225,11 @@ def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available location_config={'numeric_ids': True}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_calls_borg_with_umask_parameters(): +def test_extract_archive_calls_borg_with_umask_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', '--umask', '0770', 'repo::archive')) flexmock(module.feature).should_receive('available').and_return(True) @@ -207,10 +248,31 @@ def test_extract_archive_calls_borg_with_umask_parameters(): location_config={}, storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_calls_borg_with_lock_wait_parameters(): +def test_extract_archive_calls_borg_with_log_json_flags(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock(('borg', 'extract', '--log-json', 'repo::archive')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + + module.extract_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + location_config={}, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + + +def test_extract_archive_calls_borg_with_lock_wait_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', '--lock-wait', '5', 'repo::archive')) flexmock(module.feature).should_receive('available').and_return(True) @@ -229,6 +291,7 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters(): location_config={}, storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -252,10 +315,11 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters(): +def test_extract_archive_with_log_debug_calls_borg_with_debug_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock( ('borg', 'extract', '--debug', '--list', '--show-rc', 'repo::archive') @@ -277,6 +341,7 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -299,6 +364,7 @@ def test_extract_archive_calls_borg_with_dry_run_parameter(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -321,6 +387,7 @@ def test_extract_archive_calls_borg_with_destination_path(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), destination_path='/dest', ) @@ -344,6 +411,7 @@ def test_extract_archive_calls_borg_with_strip_components(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components=5, ) @@ -377,6 +445,7 @@ def test_extract_archive_calls_borg_with_strip_components_calculated_from_all(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components='all', ) @@ -401,6 +470,7 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components='all', ) @@ -430,6 +500,7 @@ def test_extract_archive_calls_borg_with_progress_parameter(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -446,6 +517,7 @@ def test_extract_archive_with_progress_and_extract_to_stdout_raises(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, extract_to_stdout=True, ) @@ -479,6 +551,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), extract_to_stdout=True, ) == process @@ -509,4 +582,5 @@ def test_extract_archive_skips_abspath_for_remote_repository(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index 2eed4fea..112ef4c5 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -29,6 +29,7 @@ def test_display_archives_info_calls_borg_with_parameters(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -54,6 +55,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -78,6 +80,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) @@ -106,6 +109,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -130,6 +134,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) @@ -155,6 +160,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) @@ -182,6 +188,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive='archive', json=False, prefix=None, match_archives=None), ) @@ -207,6 +214,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), local_path='borg1', ) @@ -236,11 +244,41 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), remote_path='borg1', ) +def test_display_archives_info_with_log_json_calls_borg_with_log_json_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('log-json', True).and_return( + ('--log-json',) + ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--log-json', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=True), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), + ) + + def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -266,6 +304,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -294,6 +333,7 @@ def test_display_archives_info_transforms_prefix_into_match_archives_parameters( repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix='foo'), ) @@ -322,6 +362,7 @@ def test_display_archives_info_prefers_prefix_over_archive_name_format(): repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix='foo'), ) @@ -347,6 +388,7 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -375,6 +417,7 @@ def test_display_archives_with_match_archives_option_calls_borg_with_match_archi 'match_archives': 'sh:foo-*', }, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -400,6 +443,7 @@ def test_display_archives_with_match_archives_flag_calls_borg_with_match_archive repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), ) @@ -429,6 +473,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock( archive=None, json=False, prefix=None, match_archives=None, **{argument_name: 'value'} ), diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 0a7db4cc..4e3a5f7c 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -20,6 +20,7 @@ def test_make_list_command_includes_log_info(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--info', 'repo') @@ -36,6 +37,7 @@ def test_make_list_command_includes_json_but_not_info(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -52,6 +54,7 @@ def test_make_list_command_includes_log_debug(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') @@ -68,6 +71,7 @@ def test_make_list_command_includes_json_but_not_debug(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -83,11 +87,28 @@ def test_make_list_command_includes_json(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') +def test_make_list_command_includes_log_json(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(('--log-json',)) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_list_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=True), + ) + + assert command == ('borg', 'list', '--log-json', 'repo') + + def test_make_list_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') @@ -100,6 +121,7 @@ def test_make_list_command_includes_lock_wait(): storage_config={'lock_wait': 5}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') @@ -117,6 +139,7 @@ def test_make_list_command_includes_archive(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', 'repo::archive') @@ -134,6 +157,7 @@ def test_make_list_command_includes_archive_and_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', 'repo::archive', 'var/lib') @@ -149,6 +173,7 @@ def test_make_list_command_includes_local_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), local_path='borg2', ) @@ -156,9 +181,13 @@ def test_make_list_command_includes_local_path(): def test_make_list_command_includes_remote_path(): - flexmock(module.flags).should_receive('make_flags').and_return( - ('--remote-path', 'borg2') - ).and_return(()) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args( + 'remote-path', 'borg2' + ).and_return(('--remote-path', 'borg2')) + flexmock(module.flags).should_receive('make_flags').with_args('log-json', True).and_return( + ('--log-json') + ) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -167,6 +196,7 @@ def test_make_list_command_includes_remote_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), remote_path='borg2', ) @@ -183,6 +213,7 @@ def test_make_list_command_includes_short(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False, short=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--short', 'repo') @@ -221,6 +252,7 @@ def test_make_list_command_includes_additional_flags(argument_name): format=None, **{argument_name: 'value'}, ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') @@ -263,10 +295,11 @@ def test_capture_archive_listing_does_not_raise(): archive='archive', storage_config=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), ) -def test_list_archive_calls_borg_with_parameters(): +def test_list_archive_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.logger).answer = lambda message: None @@ -281,6 +314,7 @@ def test_list_archive_calls_borg_with_parameters(): first=None, last=None, ) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -288,6 +322,7 @@ def test_list_archive_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -305,6 +340,7 @@ def test_list_archive_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, ) @@ -322,6 +358,7 @@ def test_list_archive_with_archive_and_json_errors(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -340,6 +377,7 @@ def test_list_archive_calls_borg_with_local_path(): first=None, last=None, ) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -347,6 +385,7 @@ def test_list_archive_calls_borg_with_local_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg2', remote_path=None, ).and_return(('borg2', 'list', 'repo::archive')) @@ -364,6 +403,7 @@ def test_list_archive_calls_borg_with_local_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg2', ) @@ -413,6 +453,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -431,6 +472,7 @@ def test_list_archive_calls_borg_with_archive(): first=None, last=None, ) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -438,6 +480,7 @@ def test_list_archive_calls_borg_with_archive(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -455,6 +498,7 @@ def test_list_archive_calls_borg_with_archive(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, ) @@ -485,6 +529,7 @@ def test_list_archive_without_archive_delegates_to_list_repository(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -515,6 +560,7 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -534,6 +580,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.logger).answer = lambda message: None + global_arguments = flexmock(log_json=False) default_filter_flags = { 'prefix': None, 'match_archives': None, @@ -553,6 +600,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( list_arguments=argparse.Namespace( archive='archive', paths=None, json=False, find_paths=None, **default_filter_flags ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -572,6 +620,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( list_arguments=argparse.Namespace( archive='archive', paths=None, json=False, find_paths=None, **altered_filter_flags ), + global_arguments=global_arguments, ) @@ -600,6 +649,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes } altered_filter_flags = {**default_filter_flags, **{archive_filter_flag: 'foo'}} glob_paths = ('**/*foo.txt*/**',) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.rlist).should_receive('make_rlist_command').with_args( @@ -609,6 +659,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes rlist_arguments=argparse.Namespace( repository='repo', short=True, format=None, json=None, **altered_filter_flags ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'rlist', '--repo', 'repo')) @@ -632,6 +683,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes find_paths=['foo.txt'], **default_filter_flags, ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', '--repo', 'repo', 'archive1')) @@ -650,6 +702,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes find_paths=['foo.txt'], **default_filter_flags, ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', '--repo', 'repo', 'archive2')) @@ -683,4 +736,5 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes find_paths=['foo.txt'], **altered_filter_flags, ), + global_arguments=global_arguments, ) diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 658b2e52..cd177511 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -30,6 +30,7 @@ def test_mount_archive_calls_borg_with_required_flags(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -54,6 +55,7 @@ def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_a options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -73,6 +75,7 @@ def test_mount_archive_without_archive_calls_borg_with_repository_flags_only(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -92,6 +95,7 @@ def test_mount_archive_calls_borg_with_path_flags(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -113,6 +117,7 @@ def test_mount_archive_calls_borg_with_remote_path_flags(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -133,6 +138,27 @@ def test_mount_archive_calls_borg_with_umask_flags(): options=None, storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_mount_archive_calls_borg_with_log_json_flags(): + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + insert_execute_command_mock(('borg', 'mount', '--log-json', 'repo::archive', '/mnt')) + + module.mount_archive( + repository_path='repo', + archive='archive', + mount_point='/mnt', + paths=None, + foreground=False, + options=None, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -152,6 +178,7 @@ def test_mount_archive_calls_borg_with_lock_wait_flags(): options=None, storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -172,6 +199,7 @@ def test_mount_archive_with_log_info_calls_borg_with_info_parameter(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -192,6 +220,7 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_flags(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -217,6 +246,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): options=None, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -236,4 +266,5 @@ def test_mount_archive_calls_borg_with_options_flags(): options='super_mount', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 128bdc0a..9028a0c7 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -110,7 +110,7 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') -def test_prune_archives_calls_borg_with_parameters(): +def test_prune_archives_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -123,10 +123,11 @@ def test_prune_archives_calls_borg_with_parameters(): storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): +def test_prune_archives_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -140,10 +141,11 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): dry_run=False, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): +def test_prune_archives_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -157,10 +159,11 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): dry_run=False, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): +def test_prune_archives_with_dry_run_calls_borg_with_dry_run_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -173,6 +176,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): dry_run=True, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -189,11 +193,12 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) -def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -206,11 +211,12 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters( storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) -def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_output_log_level(): +def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -223,11 +229,12 @@ def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_ou storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stats=True, ) -def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_output_log_level(): +def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -240,11 +247,12 @@ def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_out storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), list_archives=True, ) -def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): +def test_prune_archives_with_umask_calls_borg_with_umask_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'umask': '077'} @@ -258,10 +266,28 @@ def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(PRUNE_COMMAND + ('--log-json', 'repo'), logging.INFO) + + module.prune_archives( + dry_run=False, + repository_path='repo', + storage_config={}, + retention_config=flexmock(), + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + + +def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} @@ -275,6 +301,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -291,4 +318,5 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): storage_config={'extra_borg_options': {'prune': '--extra --options'}}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index 4da04dff..2f71a8ff 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -48,6 +48,7 @@ def test_create_repository_calls_borg_with_flags(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -68,6 +69,7 @@ def test_create_repository_with_dry_run_skips_borg_call(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -92,6 +94,7 @@ def test_create_repository_raises_for_borg_rcreate_error(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -111,6 +114,7 @@ def test_create_repository_skips_creation_when_repository_already_exists(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -126,6 +130,7 @@ def test_create_repository_raises_for_unknown_rinfo_command_error(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -146,6 +151,7 @@ def test_create_repository_with_source_repository_calls_borg_with_other_repo_fla repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', source_repository='other.borg', ) @@ -167,6 +173,7 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', copy_crypt_key=True, ) @@ -188,6 +195,7 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', append_only=True, ) @@ -209,6 +217,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', storage_quota='5G', ) @@ -230,6 +239,7 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', make_parent_dirs=True, ) @@ -252,6 +262,7 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -273,6 +284,49 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), + encryption_mode='repokey', + ) + + +def test_create_repository_with_log_json_calls_borg_with_log_json_flag(): + insert_rinfo_command_not_found_mock() + insert_rcreate_command_mock(RCREATE_COMMAND + ('--log-json', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=True), + encryption_mode='repokey', + ) + + +def test_create_repository_with_lock_wait_calls_borg_with_lock_wait_flag(): + insert_rinfo_command_not_found_mock() + insert_rcreate_command_mock(RCREATE_COMMAND + ('--lock-wait', '5', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + storage_config={'lock_wait': 5}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -293,6 +347,7 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', local_path='borg1', ) @@ -314,6 +369,7 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', remote_path='borg1', ) @@ -335,5 +391,6 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options repository_path='repo', storage_config={'extra_borg_options': {'rcreate': '--extra --options'}}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index 979b253e..a6e3f08c 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -7,7 +7,7 @@ from borgmatic.borg import rinfo as module from ..test_verbosity import insert_logging_mock -def test_display_repository_info_calls_borg_with_parameters(): +def test_display_repository_info_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -30,6 +30,7 @@ def test_display_repository_info_calls_borg_with_parameters(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) @@ -51,10 +52,11 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) -def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): +def test_display_repository_info_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -77,6 +79,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) @@ -102,12 +105,13 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), + global_arguments=flexmock(log_json=False), ) assert json_output == '[]' -def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter(): +def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -131,6 +135,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter( storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) @@ -156,12 +161,13 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), + global_arguments=flexmock(log_json=False), ) assert json_output == '[]' -def test_display_repository_info_with_json_calls_borg_with_json_parameter(): +def test_display_repository_info_with_json_calls_borg_with_json_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -182,6 +188,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_parameter(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), + global_arguments=flexmock(log_json=False), ) assert json_output == '[]' @@ -210,11 +217,12 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), local_path='borg1', ) -def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -237,11 +245,39 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_pa storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), remote_path='borg1', ) -def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'rinfo', '--log-json', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_repository_info( + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=True), + ) + + +def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} @@ -265,4 +301,5 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parame storage_config=storage_config, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index b83ba615..e1e04f8b 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -20,12 +20,18 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name(): archive = 'myhost-2030-01-01T14:41:17.647620' assert ( - module.resolve_archive_name('repo', archive, storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + archive, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == archive ) -def test_resolve_archive_name_calls_borg_with_parameters(): +def test_resolve_archive_name_calls_borg_with_flags(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -34,12 +40,18 @@ def test_resolve_archive_name_calls_borg_with_parameters(): ).and_return(expected_archive + '\n') assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == expected_archive ) -def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter(): +def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -49,12 +61,18 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter(): insert_logging_mock(logging.INFO) assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == expected_archive ) -def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter(): +def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -64,7 +82,13 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter( insert_logging_mock(logging.DEBUG) assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == expected_archive ) @@ -79,13 +103,18 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): assert ( module.resolve_archive_name( - 'repo', 'latest', storage_config={}, local_borg_version='1.2.3', local_path='borg1' + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + local_path='borg1', ) == expected_archive ) -def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -95,7 +124,12 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param assert ( module.resolve_archive_name( - 'repo', 'latest', storage_config={}, local_borg_version='1.2.3', remote_path='borg1' + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + remote_path='borg1', ) == expected_archive ) @@ -109,10 +143,37 @@ def test_resolve_archive_name_without_archives_raises(): ).and_return('') with pytest.raises(ValueError): - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) -def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): + expected_archive = 'archive-name' + + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ('borg', 'list', '--log-json') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, + ).and_return(expected_archive + '\n') + + assert ( + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + == expected_archive + ) + + +def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') @@ -123,7 +184,11 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter assert ( module.resolve_archive_name( - 'repo', 'latest', storage_config={'lock_wait': 'okay'}, local_borg_version='1.2.3' + 'repo', + 'latest', + storage_config={'lock_wait': 'okay'}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) == expected_archive ) @@ -145,6 +210,7 @@ def test_make_rlist_command_includes_log_info(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--info', 'repo') @@ -166,6 +232,7 @@ def test_make_rlist_command_includes_json_but_not_info(): rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -187,6 +254,7 @@ def test_make_rlist_command_includes_log_debug(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') @@ -208,6 +276,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -228,11 +297,35 @@ def test_make_rlist_command_includes_json(): rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') +def test_make_rlist_command_includes_log_json(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( + ('--log-json',) + ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), + global_arguments=flexmock(log_json=True), + ) + + assert command == ('borg', 'list', '--log-json', 'repo') + + def test_make_rlist_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') @@ -250,6 +343,7 @@ def test_make_rlist_command_includes_lock_wait(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') @@ -270,6 +364,7 @@ def test_make_rlist_command_includes_local_path(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), local_path='borg2', ) @@ -293,6 +388,7 @@ def test_make_rlist_command_includes_remote_path(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), remote_path='borg2', ) @@ -314,6 +410,7 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') @@ -332,6 +429,7 @@ def test_make_rlist_command_prefers_prefix_over_archive_name_format(): storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') @@ -352,6 +450,7 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives() rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') @@ -372,6 +471,7 @@ def test_make_rlist_command_includes_short(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None, short=True ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--short', 'repo') @@ -413,12 +513,13 @@ def test_make_rlist_command_includes_additional_flags(argument_name): format=None, **{argument_name: 'value'}, ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') -def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_parameters(): +def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_flags(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( None, None, '1.2.3' @@ -444,15 +545,17 @@ def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_p find_paths=None, format=None, ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'foo-*', 'repo') -def test_list_repository_calls_borg_with_parameters(): +def test_list_repository_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER rlist_arguments = argparse.Namespace(json=False) + global_arguments = flexmock() flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( @@ -460,6 +563,7 @@ def test_list_repository_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'rlist', 'repo')) @@ -476,6 +580,7 @@ def test_list_repository_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, ) @@ -483,6 +588,7 @@ def test_list_repository_with_json_returns_borg_output(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER rlist_arguments = argparse.Namespace(json=True) + global_arguments = flexmock() json_output = flexmock() flexmock(module.feature).should_receive('available').and_return(False) @@ -491,6 +597,7 @@ def test_list_repository_with_json_returns_borg_output(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'rlist', 'repo')) @@ -503,6 +610,7 @@ def test_list_repository_with_json_returns_borg_output(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, ) == json_output ) diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 8f41bf5a..e85729ea 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -32,6 +32,7 @@ def test_transfer_archives_calls_borg_with_flags(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -62,6 +63,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -89,6 +91,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -117,6 +120,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -146,6 +150,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): transfer_arguments=flexmock( archive='archive', progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -175,6 +180,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -204,6 +210,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -231,6 +238,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), local_path='borg2', ) @@ -262,10 +270,42 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), remote_path='borg2', ) +def test_transfer_archives_with_log_json_calls_borg_with_log_json_flags(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('log-json', True).and_return( + ('--log-json',) + ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--log-json', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, + borg_local_path='borg', + extra_environment=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + global_arguments=flexmock(log_json=True), + ) + + def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -294,6 +334,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -321,6 +362,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): transfer_arguments=flexmock( archive=None, progress=True, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -356,6 +398,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): source_repository=None, **{argument_name: 'value'}, ), + global_arguments=flexmock(log_json=False), ) @@ -385,4 +428,5 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository='other' ), + global_arguments=flexmock(log_json=False), ) From 403ae0f698d8f612832912e32f778f9245eff781 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 9 May 2023 10:14:03 -0700 Subject: [PATCH 186/344] Clarify configuration comment about source_directories also accepting files (#693). --- borgmatic/config/schema.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 0cf02b25..903c0432 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -30,8 +30,8 @@ properties: items: type: string description: | - List of source directories to backup. Globs and tildes are - expanded. Do not backslash spaces in path names. + List of source directories and files to backup. Globs and + tildes are expanded. Do not backslash spaces in path names. example: - /home - /etc From 62b11ba16b974511e9505dea90822b7524538669 Mon Sep 17 00:00:00 2001 From: ennui Date: Sat, 13 May 2023 11:20:47 +0000 Subject: [PATCH 187/344] Docs: add Gentoo Linux to other ways to install --- docs/how-to/set-up-backups.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index de5bf8b9..d428ddc3 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -87,6 +87,7 @@ installing borgmatic: * [Debian](https://tracker.debian.org/pkg/borgmatic) * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic) * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic) + * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic) * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) From c6126a9226f223ea2fbf41667cfbf5b9550a024b Mon Sep 17 00:00:00 2001 From: ennui Date: Sat, 13 May 2023 11:22:47 +0000 Subject: [PATCH 188/344] Docs: add Gentoo Linux to other ways to install --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index d428ddc3..515f017e 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -87,8 +87,8 @@ installing borgmatic: * [Debian](https://tracker.debian.org/pkg/borgmatic) * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic) * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic) - * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic) * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) + * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic) * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) * [OpenBSD](https://openports.pl/path/sysutils/borgmatic) From 8eb05b840a9a07655f0652feb39a15822bcbd3a4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 14 May 2023 09:59:28 -0700 Subject: [PATCH 189/344] Log a warning when "borgmatic borg" is run with an action that borgmatic natively supports (#694). --- NEWS | 2 +- borgmatic/borg/borg.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 17560d7e..ac555822 100644 --- a/NEWS +++ b/NEWS @@ -18,7 +18,7 @@ --editable" development installs. * #691: Fix error in "borgmatic restore" action when the configured repository path is relative instead of absolute. - * Run "borgmatic borg" action without capturing output so interactive prompts and flags like + * #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like "--progress" still work. 1.7.12 diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index 1c41b8ec..82fecc1a 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -1,5 +1,6 @@ import logging +import borgmatic.commands.arguments import borgmatic.logger from borgmatic.borg import environment, flags from borgmatic.execute import DO_NOT_CAPTURE, execute_command @@ -36,6 +37,14 @@ def run_arbitrary_borg( command_options_start_index = 2 if options[0] in BORG_SUBCOMMANDS_WITH_SUBCOMMANDS else 1 borg_command = tuple(options[:command_options_start_index]) command_options = tuple(options[command_options_start_index:]) + + if ( + borg_command + and borg_command[0] in borgmatic.commands.arguments.SUBPARSER_ALIASES.keys() + ): + logger.warning( + f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}" + ) except IndexError: borg_command = () command_options = () From 1bc7bb49714a33ac9613a2da10cc7951ccace276 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 15 May 2023 23:04:42 +0530 Subject: [PATCH 190/344] feat: store configs used to create an archive in the archive --- borgmatic/borg/create.py | 2 +- borgmatic/commands/borgmatic.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index e3b70eb5..284789d6 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -351,7 +351,7 @@ def create_archive( sources = deduplicate_directories( map_directories_to_devices( expand_directories( - tuple(location_config.get('source_directories', ())) + borgmatic_source_directories + tuple(location_config.get('source_directories', ())) + borgmatic_source_directories + tuple(global_arguments.config_paths) ) ), additional_directory_devices=map_directories_to_devices( diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 44396cd4..3afa625b 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -45,10 +45,10 @@ logger = logging.getLogger(__name__) LEGACY_CONFIG_PATH = '/etc/borgmatic/config' -def run_configuration(config_filename, config, arguments): +def run_configuration(config_filename, config, arguments, used_config_paths): ''' - Given a config filename, the corresponding parsed config dict, and command-line arguments as a - dict from subparser name to a namespace of parsed arguments, execute the defined create, prune, + Given a config filename, the corresponding parsed config dict, command-line arguments as a + dict from subparser name to a namespace of parsed arguments, and a list of paths of all configs used, execute the defined create, prune, compact, check, and/or other actions. Yield a combination of: @@ -61,6 +61,7 @@ def run_configuration(config_filename, config, arguments): for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks') ) global_arguments = arguments['global'] + global_arguments.config_paths = used_config_paths local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') @@ -644,8 +645,9 @@ def collect_configuration_run_summary_logs(configs, arguments): # Execute the actions corresponding to each configuration file. json_results = [] + used_config_paths = list(configs.keys()) for config_filename, config in configs.items(): - results = list(run_configuration(config_filename, config, arguments)) + results = list(run_configuration(config_filename, config, arguments, used_config_paths)) error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord)) if error_logs: From 49b4d371cea63afa872e7e81f4e95666567ae179 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 16 May 2023 00:24:19 +0530 Subject: [PATCH 191/344] create and add content to borgmatic-manifest.json --- borgmatic/actions/create.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index a3f8da57..10a35ca4 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -1,4 +1,5 @@ import json +import os import logging import borgmatic.borg.create @@ -7,9 +8,30 @@ import borgmatic.hooks.command import borgmatic.hooks.dispatch import borgmatic.hooks.dump +from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY + logger = logging.getLogger(__name__) +def create_borgmatic_manifest(location, config_paths, dry_run): + ''' + Create a borgmatic manifest file to store the paths to the configuration files used to create + the archive. + ''' + if dry_run: + return + + borgmatic_source_directory = location.get('borgmatic_source_directory') if location.get('borgmatic_source_directory') else DEFAULT_BORGMATIC_SOURCE_DIRECTORY + + borgmatic_manifest_path = os.path.expanduser(os.path.join(borgmatic_source_directory, 'bootstrap', 'configs-list.json')) + + if not os.path.exists(borgmatic_manifest_path): + os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True) + + with open(borgmatic_manifest_path, 'w') as f: + json.dump(config_paths, f) + + def run_create( config_filename, repository, @@ -59,6 +81,7 @@ def run_create( location, global_arguments.dry_run, ) + create_borgmatic_manifest(location, global_arguments.config_paths, global_arguments.dry_run) stream_processes = [process for processes in active_dumps.values() for process in processes] json_output = borgmatic.borg.create.create_archive( From 645d29b040c58dce7d971df84ab40a2e1678bb6a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 15 May 2023 23:17:45 -0700 Subject: [PATCH 192/344] Fix archive checks being skipped even when particular archives haven't been checked recently (#688). --- NEWS | 3 + borgmatic/borg/check.py | 230 ++++++-- docs/how-to/deal-with-very-large-backups.md | 1 + docs/how-to/make-per-application-backups.md | 3 + tests/unit/borg/test_borg.py | 18 +- tests/unit/borg/test_check.py | 602 +++++++++++++------- 6 files changed, 587 insertions(+), 270 deletions(-) diff --git a/NEWS b/NEWS index ac555822..bcfe22b2 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,9 @@ https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install --editable" development installs. + * #688: Fix archive checks being skipped even when particular archives haven't been checked + recently. This occurred when using multiple borgmatic configuration files with different + "archive_name_format"s, for instance. * #691: Fix error in "borgmatic restore" action when the configured repository path is relative instead of absolute. * #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 52d5208c..63acbe26 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -1,5 +1,7 @@ import argparse import datetime +import hashlib +import itertools import json import logging import os @@ -88,12 +90,18 @@ def parse_frequency(frequency): def filter_checks_on_frequency( - location_config, consistency_config, borg_repository_id, checks, force + location_config, + consistency_config, + borg_repository_id, + checks, + force, + archives_check_id=None, ): ''' Given a location config, a consistency config with a "checks" sequence of dicts, a Borg - repository ID, a sequence of checks, and whether to force checks to run, filter down those - checks based on the configured "frequency" for each check as compared to its check time file. + repository ID, a sequence of checks, whether to force checks to run, and an ID for the archives + check potentially being run (if any), filter down those checks based on the configured + "frequency" for each check as compared to its check time file. In other words, a check whose check time file's timestamp is too new (based on the configured frequency) will get cut from the returned sequence of checks. Example: @@ -127,8 +135,8 @@ def filter_checks_on_frequency( if not frequency_delta: continue - check_time = read_check_time( - make_check_time_path(location_config, borg_repository_id, check) + check_time = probe_for_check_time( + location_config, borg_repository_id, check, archives_check_id ) if not check_time: continue @@ -145,36 +153,19 @@ def filter_checks_on_frequency( return tuple(filtered_checks) -def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None): +def make_archive_filter_flags( + local_borg_version, storage_config, checks, check_last=None, prefix=None +): ''' Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the check last value, and a consistency check prefix, transform the checks into tuple of - command-line flags. + command-line flags for filtering archives in a check command. - For example, given parsed checks of: - - ('repository',) - - This will be returned as: - - ('--repository-only',) - - However, if both "repository" and "archives" are in checks, then omit them from the returned - flags because Borg does both checks by default. If "data" is in checks, that implies "archives". - - Additionally, if a check_last value is given and "archives" is in checks, then include a - "--last" flag. And if a prefix value is given and "archives" is in checks, then include a - "--match-archives" flag. + If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if + a prefix value is given and "archives" is in checks, then include a "--match-archives" flag. ''' - if 'data' in checks: - data_flags = ('--verify-data',) - checks += ('archives',) - else: - data_flags = () - - if 'archives' in checks: - last_flags = ('--last', str(check_last)) if check_last else () - match_archives_flags = ( + if 'archives' in checks or 'data' in checks: + return (('--last', str(check_last)) if check_last else ()) + ( ( ('--match-archives', f'sh:{prefix}*') if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) @@ -189,19 +180,53 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None ) ) ) - else: - last_flags = () - match_archives_flags = () - if check_last: - logger.warning( - 'Ignoring check_last option, as "archives" or "data" are not in consistency checks' - ) - if prefix: - logger.warning( - 'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks' - ) - common_flags = last_flags + match_archives_flags + data_flags + if check_last: + logger.warning( + 'Ignoring check_last option, as "archives" or "data" are not in consistency checks' + ) + if prefix: + logger.warning( + 'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks' + ) + + return () + + +def make_archives_check_id(archive_filter_flags): + ''' + Given a sequence of flags to filter archives, return a unique hash corresponding to those + particular flags. If there are no flags, return None. + ''' + if not archive_filter_flags: + return None + + return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest() + + +def make_check_flags(checks, archive_filter_flags): + ''' + Given a parsed sequence of checks and a sequence of flags to filter archives, transform the + checks into tuple of command-line check flags. + + For example, given parsed checks of: + + ('repository',) + + This will be returned as: + + ('--repository-only',) + + However, if both "repository" and "archives" are in checks, then omit them from the returned + flags because Borg does both checks by default. If "data" is in checks, that implies "archives". + ''' + if 'data' in checks: + data_flags = ('--verify-data',) + checks += ('archives',) + else: + data_flags = () + + common_flags = archive_filter_flags + data_flags if {'repository', 'archives'}.issubset(set(checks)): return common_flags @@ -212,18 +237,27 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None ) -def make_check_time_path(location_config, borg_repository_id, check_type): +def make_check_time_path(location_config, borg_repository_id, check_type, archives_check_id=None): ''' - Given a location configuration dict, a Borg repository ID, and the name of a check type - ("repository", "archives", etc.), return a path for recording that check's time (the time of - that check last occurring). + Given a location configuration dict, a Borg repository ID, the name of a check type + ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a + path for recording that check's time (the time of that check last occurring). ''' + borgmatic_source_directory = os.path.expanduser( + location_config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY) + ) + + if check_type in ('archives', 'data'): + return os.path.join( + borgmatic_source_directory, + 'checks', + borg_repository_id, + check_type, + archives_check_id if archives_check_id else 'all', + ) + return os.path.join( - os.path.expanduser( - location_config.get( - 'borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY - ) - ), + borgmatic_source_directory, 'checks', borg_repository_id, check_type, @@ -253,6 +287,74 @@ def read_check_time(path): return None +def probe_for_check_time(location_config, borg_repository_id, check, archives_check_id): + ''' + Given a location configuration dict, a Borg repository ID, the name of a check type + ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a + the corresponding check time or None if such a check time does not exist. + + When the check type is "archives" or "data", this function probes two different paths to find + the check time, e.g.: + + ~/.borgmatic/checks/1234567890/archives/9876543210 + ~/.borgmatic/checks/1234567890/archives/all + + ... and returns the modification time of the first file found (if any). The first path + represents a more specific archives check time (a check on a subset of archives), and the second + is a fallback to the last "all" archives check. + + For other check types, this function reads from a single check time path, e.g.: + + ~/.borgmatic/checks/1234567890/repository + ''' + check_times = ( + read_check_time(group[0]) + for group in itertools.groupby( + ( + make_check_time_path(location_config, borg_repository_id, check, archives_check_id), + make_check_time_path(location_config, borg_repository_id, check), + ) + ) + ) + + try: + return next(check_time for check_time in check_times if check_time) + except StopIteration: + return None + + +def upgrade_check_times(location_config, borg_repository_id): + ''' + Given a location configuration dict and a Borg repository ID, upgrade any corresponding check + times on disk from old-style paths to new-style paths. + + Currently, the only upgrade performed is renaming an archive or data check path that looks like: + + ~/.borgmatic/checks/1234567890/archives + + to: + + ~/.borgmatic/checks/1234567890/archives/all + ''' + for check_type in ('archives', 'data'): + new_path = make_check_time_path(location_config, borg_repository_id, check_type, 'all') + old_path = os.path.dirname(new_path) + temporary_path = f'{old_path}.temp' + + if not os.path.isfile(old_path) and not os.path.isfile(temporary_path): + return + + logger.debug(f'Upgrading archives check time from {old_path} to {new_path}') + + try: + os.rename(old_path, temporary_path) + except FileNotFoundError: + pass + + os.mkdir(old_path) + os.rename(temporary_path, new_path) + + def check_archives( repository_path, location_config, @@ -292,16 +394,26 @@ def check_archives( except (json.JSONDecodeError, KeyError): raise ValueError(f'Cannot determine Borg repository ID for {repository_path}') + upgrade_check_times(location_config, borg_repository_id) + + check_last = consistency_config.get('check_last', None) + prefix = consistency_config.get('prefix') + configured_checks = parse_checks(consistency_config, only_checks) + lock_wait = None + extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') + archive_filter_flags = make_archive_filter_flags( + local_borg_version, storage_config, configured_checks, check_last, prefix + ) + archives_check_id = make_archives_check_id(archive_filter_flags) + checks = filter_checks_on_frequency( location_config, consistency_config, borg_repository_id, - parse_checks(consistency_config, only_checks), + configured_checks, force, + archives_check_id, ) - check_last = consistency_config.get('check_last', None) - lock_wait = None - extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') if set(checks).intersection({'repository', 'archives', 'data'}): lock_wait = storage_config.get('lock_wait') @@ -312,12 +424,10 @@ def check_archives( if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') - prefix = consistency_config.get('prefix') - full_command = ( (local_path, 'check') + (('--repair',) if repair else ()) - + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix) + + make_check_flags(checks, archive_filter_flags) + (('--remote-path', remote_path) if remote_path else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) @@ -339,7 +449,9 @@ def check_archives( execute_command(full_command, extra_environment=borg_environment) for check in checks: - write_check_time(make_check_time_path(location_config, borg_repository_id, check)) + write_check_time( + make_check_time_path(location_config, borg_repository_id, check, archives_check_id) + ) if 'extract' in checks: extract.extract_last_archive_dry_run( diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index e5962c1e..5beb9f24 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -95,6 +95,7 @@ See [Borg's check documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html) for more information. + ### Check frequency New in version 1.6.2 You can diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index f2ddf012..7832dc43 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -81,6 +81,9 @@ If `archive_name_format` is unspecified, the default is `{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a timestamp in a particular format. + +### Achive filtering + New in version 1.7.11 borgmatic uses the `archive_name_format` option to automatically limit which archives get used for actions operating on multiple archives. This prevents, for diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 4c71ce1a..5ae013f8 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -7,7 +7,7 @@ from borgmatic.borg import borg as module from ..test_verbosity import insert_logging_mock -def test_run_arbitrary_borg_calls_borg_with_parameters(): +def test_run_arbitrary_borg_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -28,7 +28,7 @@ def test_run_arbitrary_borg_calls_borg_with_parameters(): ) -def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): +def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -50,7 +50,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): ) -def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): +def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -72,7 +72,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): ) -def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} @@ -96,7 +96,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters( ) -def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter(): +def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -142,7 +142,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): ) -def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -166,7 +166,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet ) -def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): +def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -187,7 +187,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): ) -def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): +def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -208,7 +208,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): ) -def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): +def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').never() diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 4cd6aa77..aad973bd 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -96,7 +96,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks(): module.datetime.timedelta(weeks=4) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return(None) + flexmock(module).should_receive('probe_for_check_time').and_return(None) assert module.filter_checks_on_frequency( location_config={}, @@ -104,6 +104,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks(): borg_repository_id='repo', checks=('repository', 'archives'), force=False, + archives_check_id='1234', ) == ('repository', 'archives') @@ -126,6 +127,7 @@ def test_filter_checks_on_frequency_retains_check_without_frequency(): borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == ('archives',) @@ -134,7 +136,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency(): module.datetime.timedelta(hours=1) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return( + flexmock(module).should_receive('probe_for_check_time').and_return( module.datetime.datetime(year=module.datetime.MINYEAR, month=1, day=1) ) @@ -144,6 +146,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency(): borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == ('archives',) @@ -152,7 +155,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file() module.datetime.timedelta(hours=1) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return(None) + flexmock(module).should_receive('probe_for_check_time').and_return(None) assert module.filter_checks_on_frequency( location_config={}, @@ -160,6 +163,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file() borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == ('archives',) @@ -168,7 +172,9 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency(): module.datetime.timedelta(hours=1) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return(module.datetime.datetime.now()) + flexmock(module).should_receive('probe_for_check_time').and_return( + module.datetime.datetime.now() + ) assert ( module.filter_checks_on_frequency( @@ -177,6 +183,7 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency(): borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == () ) @@ -189,32 +196,177 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_ borg_repository_id='repo', checks=('archives',), force=True, + archives_check_id='1234', ) == ('archives',) -def test_make_check_flags_with_repository_check_returns_flag(): +def test_make_archive_filter_flags_with_default_checks_and_prefix_returns_default_flags(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('repository',)) + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives'), + prefix='foo', + ) + + assert flags == ('--match-archives', 'sh:foo*') + + +def test_make_archive_filter_flags_with_all_checks_and_prefix_returns_default_flags(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', + ) + + assert flags == ('--match-archives', 'sh:foo*') + + +def test_make_archive_filter_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', + ) + + assert flags == ('--glob-archives', 'foo*') + + +def test_make_archive_filter_flags_with_archives_check_and_last_includes_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), check_last=3) + + assert flags == ('--last', '3') + + +def test_make_archive_filter_flags_with_data_check_and_last_includes_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), check_last=3) + + assert flags == ('--last', '3') + + +def test_make_archive_filter_flags_with_repository_check_and_last_omits_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), check_last=3) + + assert flags == () + + +def test_make_archive_filter_flags_with_default_checks_and_last_includes_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) + + assert flags == ('--last', '3') + + +def test_make_archive_filter_flags_with_archives_check_and_prefix_includes_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix='foo-') + + assert flags == ('--match-archives', 'sh:foo-*') + + +def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), prefix='foo-') + + assert flags == ('--match-archives', 'sh:foo-*') + + +def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + + flags = module.make_archive_filter_flags( + '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 + ) + + assert flags == ('--match-archives', 'sh:bar-*') + + +def test_make_archive_filter_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix=None) + + assert flags == () + + +def test_make_archive_filter_flags_with_repository_check_and_prefix_omits_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), prefix='foo-') + + assert flags == () + + +def test_make_archive_filter_flags_with_default_checks_and_prefix_includes_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') + + assert flags == ('--match-archives', 'sh:foo-*') + + +def test_make_archives_check_id_with_flags_returns_a_value_and_does_not_raise(): + assert module.make_archives_check_id(('--match-archives', 'sh:foo-*')) + + +def test_make_archives_check_id_with_empty_flags_returns_none(): + assert module.make_archives_check_id(()) is None + + +def test_make_check_flags_with_repository_check_returns_flag(): + flags = module.make_check_flags(('repository',), ()) assert flags == ('--repository-only',) def test_make_check_flags_with_archives_check_returns_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('archives',)) + flags = module.make_check_flags(('archives',), ()) assert flags == ('--archives-only',) +def test_make_check_flags_with_archive_filtler_flags_includes_those_flags(): + flags = module.make_check_flags(('archives',), ('--match-archives', 'sh:foo-*')) + + assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') + + def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('data',)) + flags = module.make_check_flags(('data',), ()) assert flags == ( '--archives-only', @@ -226,7 +378,7 @@ def test_make_check_flags_with_extract_omits_extract_flag(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('extract',)) + flags = module.make_check_flags(('extract',), ()) assert flags == () @@ -236,151 +388,66 @@ def test_make_check_flags_with_repository_and_data_checks_does_not_return_reposi flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', - {}, ( 'repository', 'data', ), + (), ) assert flags == ('--verify-data',) -def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags( - '1.2.3', - {}, - ('repository', 'archives'), - prefix='foo', +def test_make_check_time_path_with_borgmatic_source_directory_includes_it(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return( + '/home/user/.borgmatic' ) - assert flags == ('--match-archives', 'sh:foo*') - - -def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags( - '1.2.3', - {}, - ('repository', 'archives', 'extract'), - prefix='foo', + assert ( + module.make_check_time_path( + {'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'archives', '5678' + ) + == '/home/user/.borgmatic/checks/1234/archives/5678' ) - assert flags == ('--match-archives', 'sh:foo*') +def test_make_check_time_path_without_borgmatic_source_directory_uses_default(): + flexmock(module.os.path).should_receive('expanduser').with_args( + module.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY + ).and_return('/home/user/.borgmatic') -def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): - flexmock(module.feature).should_receive('available').and_return(False) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags( - '1.2.3', - {}, - ('repository', 'archives', 'extract'), - prefix='foo', + assert ( + module.make_check_time_path({}, '1234', 'archives', '5678') + == '/home/user/.borgmatic/checks/1234/archives/5678' ) - assert flags == ('--glob-archives', 'foo*') - -def test_make_check_flags_with_archives_check_and_last_includes_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('archives',), check_last=3) - - assert flags == ('--archives-only', '--last', '3') - - -def test_make_check_flags_with_data_check_and_last_includes_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('data',), check_last=3) - - assert flags == ('--archives-only', '--last', '3', '--verify-data') - - -def test_make_check_flags_with_repository_check_and_last_omits_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository',), check_last=3) - - assert flags == ('--repository-only',) - - -def test_make_check_flags_with_default_checks_and_last_includes_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) - - assert flags == ('--last', '3') - - -def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix='foo-') - - assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') - - -def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('data',), prefix='foo-') - - assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data') - - -def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, 'bar-{now}', '1.2.3' # noqa: FS003 - ).and_return(('--match-archives', 'sh:bar-*')) - - flags = module.make_check_flags( - '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 +def test_make_check_time_path_with_archives_check_and_no_archives_check_id_defaults_to_all(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return( + '/home/user/.borgmatic' ) - assert flags == ('--archives-only', '--match-archives', 'sh:bar-*') + assert ( + module.make_check_time_path( + {'borgmatic_source_directory': '~/.borgmatic'}, + '1234', + 'archives', + ) + == '/home/user/.borgmatic/checks/1234/archives/all' + ) -def test_make_check_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) +def test_make_check_time_path_with_repositories_check_ignores_archives_check_id(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return( + '/home/user/.borgmatic' + ) - flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix=None) - - assert flags == ('--archives-only',) - - -def test_make_check_flags_with_repository_check_and_prefix_omits_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository',), prefix='foo-') - - assert flags == ('--repository-only',) - - -def test_make_check_flags_with_default_checks_and_prefix_includes_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') - - assert flags == ('--match-archives', 'sh:foo-*') + assert ( + module.make_check_time_path( + {'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'repository', '5678' + ) + == '/home/user/.borgmatic/checks/1234/repository' + ) def test_read_check_time_does_not_raise(): @@ -395,14 +462,135 @@ def test_read_check_time_on_missing_file_does_not_raise(): assert module.read_check_time('/path') is None +def test_probe_for_check_time_uses_first_of_multiple_check_times(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(1).and_return(2) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + + +def test_probe_for_check_time_deduplicates_identical_check_time_paths(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/5678') + flexmock(module).should_receive('read_check_time').and_return(1).once() + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + + +def test_probe_for_check_time_skips_none_check_time(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(None).and_return(2) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2 + + +def test_probe_for_check_time_uses_single_check_time(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(1).and_return(None) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + + +def test_probe_for_check_time_returns_none_when_no_check_time_found(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(None).and_return(None) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) is None + + +def test_upgrade_check_times_renames_old_check_paths_to_all(): + base_path = '~/.borgmatic/checks/1234' + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'archives', 'all' + ).and_return(f'{base_path}/archives/all') + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'data', 'all' + ).and_return(f'{base_path}/data/all') + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return( + True + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/archives.temp' + ).and_return(False) + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/data.temp' + ).and_return(False) + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives', f'{base_path}/archives.temp' + ).once() + flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once() + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives.temp', f'{base_path}/archives/all' + ).once() + + module.upgrade_check_times(flexmock(), flexmock()) + + +def test_upgrade_check_times_skips_missing_check_paths(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/all' + ) + flexmock(module.os.path).should_receive('isfile').and_return(False) + flexmock(module.os).should_receive('rename').never() + flexmock(module.os).should_receive('mkdir').never() + + module.upgrade_check_times(flexmock(), flexmock()) + + +def test_upgrade_check_times_renames_stale_temporary_check_path(): + base_path = '~/.borgmatic/checks/1234' + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'archives', 'all' + ).and_return(f'{base_path}/archives/all') + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'data', 'all' + ).and_return(f'{base_path}/data/all') + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/archives.temp' + ).and_return(True) + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/data.temp' + ).and_return(False) + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives', f'{base_path}/archives.temp' + ).and_raise(FileNotFoundError) + flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once() + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives.temp', f'{base_path}/archives/all' + ).once() + + module.upgrade_check_times(flexmock(), flexmock()) + + def test_check_archives_with_progress_calls_borg_with_progress_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module).should_receive('execute_command').never() flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -429,11 +617,14 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): def test_check_archives_with_repair_calls_borg_with_repair_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module).should_receive('execute_command').never() flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -469,18 +660,15 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): def test_check_archives_calls_borg_with_parameters(checks): check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - {}, - checks, - check_last, - prefix=None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -500,11 +688,14 @@ def test_check_archives_with_json_error_raises(): checks = ('archives',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"unexpected": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) with pytest.raises(ValueError): module.check_archives( @@ -521,9 +712,12 @@ def test_check_archives_with_missing_json_keys_raises(): checks = ('archives',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return('{invalid JSON') + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) with pytest.raises(ValueError): module.check_archives( @@ -540,11 +734,14 @@ def test_check_archives_with_extract_check_calls_extract_only(): checks = ('extract',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').never() flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.extract).should_receive('extract_last_archive_dry_run').once() @@ -564,11 +761,14 @@ def test_check_archives_with_extract_check_calls_extract_only(): def test_check_archives_with_log_info_calls_borg_with_info_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_logging_mock(logging.INFO) @@ -589,11 +789,14 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_logging_mock(logging.DEBUG) @@ -613,11 +816,14 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): def test_check_archives_without_any_checks_bails(): consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(()) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(()) insert_execute_command_never() module.check_archives( @@ -634,18 +840,15 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): checks = ('repository',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - {}, - checks, - check_last, - prefix=None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1', 'check', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -666,18 +869,15 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( checks = ('repository',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - {}, - checks, - check_last, - prefix=None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -699,18 +899,15 @@ def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): check_last = flexmock() storage_config = {} consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - storage_config, - checks, - check_last, - None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--log-json', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -731,18 +928,15 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): check_last = flexmock() storage_config = {'lock_wait': 5} consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - storage_config, - checks, - check_last, - None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -763,14 +957,15 @@ def test_check_archives_with_retention_prefix(): check_last = flexmock() prefix = 'foo-' consistency_config = {'check_last': check_last, 'prefix': prefix} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', {}, checks, check_last, prefix - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -789,11 +984,14 @@ def test_check_archives_with_retention_prefix(): def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--extra', '--options', 'repo')) From ba845d40081c31e646faa2add2b57c12e3811e24 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 15 May 2023 23:25:13 -0700 Subject: [PATCH 193/344] Codespell saves the day. --- docs/how-to/make-per-application-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 7832dc43..b565d668 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -82,7 +82,7 @@ If `archive_name_format` is unspecified, the default is timestamp in a particular format. -### Achive filtering +### Archive filtering New in version 1.7.11 borgmatic uses the `archive_name_format` option to automatically limit which archives From b10148844bc0214f07e0e9357c22d82c8262790d Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 16 May 2023 14:00:23 +0530 Subject: [PATCH 194/344] change config_paths var name to used_config_paths to avoid collisions --- borgmatic/actions/create.py | 28 ++++++++++++++++++++++------ borgmatic/borg/create.py | 4 +++- borgmatic/commands/borgmatic.py | 11 +++++------ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 10a35ca4..2dba94ba 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -2,6 +2,8 @@ import json import os import logging +import importlib_metadata + import borgmatic.borg.create import borgmatic.config.validate import borgmatic.hooks.command @@ -17,19 +19,31 @@ def create_borgmatic_manifest(location, config_paths, dry_run): ''' Create a borgmatic manifest file to store the paths to the configuration files used to create the archive. - ''' + ''' if dry_run: return - - borgmatic_source_directory = location.get('borgmatic_source_directory') if location.get('borgmatic_source_directory') else DEFAULT_BORGMATIC_SOURCE_DIRECTORY - borgmatic_manifest_path = os.path.expanduser(os.path.join(borgmatic_source_directory, 'bootstrap', 'configs-list.json')) + borgmatic_source_directory = ( + location.get('borgmatic_source_directory') + if location.get('borgmatic_source_directory') + else DEFAULT_BORGMATIC_SOURCE_DIRECTORY + ) + + borgmatic_manifest_path = os.path.expanduser( + os.path.join(borgmatic_source_directory, 'bootstrap', 'configs-list.json') + ) if not os.path.exists(borgmatic_manifest_path): os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True) with open(borgmatic_manifest_path, 'w') as f: - json.dump(config_paths, f) + json.dump( + { + 'borgmatic_version': importlib_metadata.version('borgmatic'), + 'config_paths': config_paths, + }, + f, + ) def run_create( @@ -81,7 +95,9 @@ def run_create( location, global_arguments.dry_run, ) - create_borgmatic_manifest(location, global_arguments.config_paths, global_arguments.dry_run) + create_borgmatic_manifest( + location, global_arguments.used_config_paths, global_arguments.dry_run + ) stream_processes = [process for processes in active_dumps.values() for process in processes] json_output = borgmatic.borg.create.create_archive( diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 284789d6..618376a5 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -351,7 +351,9 @@ def create_archive( sources = deduplicate_directories( map_directories_to_devices( expand_directories( - tuple(location_config.get('source_directories', ())) + borgmatic_source_directories + tuple(global_arguments.config_paths) + tuple(location_config.get('source_directories', ())) + + borgmatic_source_directories + + tuple(global_arguments.used_config_paths) ) ), additional_directory_devices=map_directories_to_devices( diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 3afa625b..1eb68349 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -45,10 +45,10 @@ logger = logging.getLogger(__name__) LEGACY_CONFIG_PATH = '/etc/borgmatic/config' -def run_configuration(config_filename, config, arguments, used_config_paths): +def run_configuration(config_filename, config, arguments): ''' - Given a config filename, the corresponding parsed config dict, command-line arguments as a - dict from subparser name to a namespace of parsed arguments, and a list of paths of all configs used, execute the defined create, prune, + Given a config filename, the corresponding parsed config dict, and command-line arguments as a + dict from subparser name to a namespace of parsed arguments, execute the defined create, prune, compact, check, and/or other actions. Yield a combination of: @@ -61,7 +61,6 @@ def run_configuration(config_filename, config, arguments, used_config_paths): for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks') ) global_arguments = arguments['global'] - global_arguments.config_paths = used_config_paths local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') @@ -645,9 +644,8 @@ def collect_configuration_run_summary_logs(configs, arguments): # Execute the actions corresponding to each configuration file. json_results = [] - used_config_paths = list(configs.keys()) for config_filename, config in configs.items(): - results = list(run_configuration(config_filename, config, arguments, used_config_paths)) + results = list(run_configuration(config_filename, config, arguments)) error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord)) if error_logs: @@ -729,6 +727,7 @@ def main(): # pragma: no cover sys.exit(0) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) + global_arguments.used_config_paths = list(config_filenames) configs, parse_logs = load_configurations( config_filenames, global_arguments.overrides, global_arguments.resolve_env ) From b45e45f1615897c3c60c7433f92fe37b049c1e07 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 16 May 2023 09:36:50 -0700 Subject: [PATCH 195/344] Partial conversion of showing repository labels in logs instead of paths (part of #635). --- borgmatic/actions/borg.py | 4 +++- borgmatic/actions/break_lock.py | 4 +++- borgmatic/actions/check.py | 2 +- borgmatic/actions/compact.py | 8 ++++++-- borgmatic/actions/create.py | 2 +- borgmatic/actions/extract.py | 4 +++- borgmatic/actions/info.py | 4 +++- borgmatic/actions/list.py | 4 ++-- borgmatic/actions/mount.py | 6 ++++-- borgmatic/actions/prune.py | 2 +- borgmatic/actions/rcreate.py | 2 +- borgmatic/actions/restore.py | 16 +++++++++------- borgmatic/actions/rinfo.py | 4 +++- borgmatic/actions/rlist.py | 2 +- borgmatic/actions/transfer.py | 4 +++- borgmatic/commands/borgmatic.py | 15 ++++++++++----- 16 files changed, 54 insertions(+), 29 deletions(-) diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index ec445fbb..44ffc951 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -22,7 +22,9 @@ def run_borg( if borg_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, borg_arguments.repository ): - logger.info(f'{repository["path"]}: Running arbitrary Borg command') + logger.info( + f'{repository.get("label", repository["path"])}: Running arbitrary Borg command' + ) archive_name = borgmatic.borg.rlist.resolve_archive_name( repository['path'], borg_arguments.archive, diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index f049e772..a00d5785 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -21,7 +21,9 @@ def run_break_lock( if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, break_lock_arguments.repository ): - logger.info(f'{repository["path"]}: Breaking repository and cache locks') + logger.info( + f'{repository.get("label", repository["path"])}: Breaking repository and cache locks' + ) borgmatic.borg.break_lock.break_lock( repository['path'], storage, diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index aac536e3..610d41ee 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -37,7 +37,7 @@ def run_check( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository["path"]}: Running consistency checks') + logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks') borgmatic.borg.check.check_archives( repository['path'], location, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 24b30c0e..ad680d21 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -39,7 +39,9 @@ def run_compact( **hook_context, ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): - logger.info(f'{repository["path"]}: Compacting segments{dry_run_label}') + logger.info( + f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}' + ) borgmatic.borg.compact.compact_segments( global_arguments.dry_run, repository['path'], @@ -53,7 +55,9 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover - logger.info(f'{repository["path"]}: Skipping compact (only available/needed in Borg 1.2+)') + logger.info( + f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)' + ) borgmatic.hooks.command.execute_hook( hooks.get('after_compact'), hooks.get('umask'), diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index a3f8da57..cb8b1cf4 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -42,7 +42,7 @@ def run_create( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository["path"]}: Creating archive{dry_run_label}') + logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}') borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 1f4317cd..0bb1efb7 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -35,7 +35,9 @@ def run_extract( if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): - logger.info(f'{repository["path"]}: Extracting archive {extract_arguments.archive}') + logger.info( + f'{repository.get("label", repository["path"])}: Extracting archive {extract_arguments.archive}' + ) borgmatic.borg.extract.extract_archive( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index d138dbd4..91699623 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -26,7 +26,9 @@ def run_info( repository, info_arguments.repository ): if not info_arguments.json: # pragma: nocover - logger.answer(f'{repository["path"]}: Displaying archive summary information') + logger.answer( + f'{repository.get("label", repository["path"])}: Displaying archive summary information' + ) info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( repository['path'], info_arguments.archive, diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 548f1979..720fab1c 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -26,9 +26,9 @@ def run_list( ): if not list_arguments.json: # pragma: nocover if list_arguments.find_paths: - logger.answer(f'{repository["path"]}: Searching archives') + logger.answer(f'{repository.get("label", repository["path"])}: Searching archives') elif not list_arguments.archive: - logger.answer(f'{repository["path"]}: Listing archives') + logger.answer(f'{repository.get("label", repository["path"])}: Listing archives') list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( repository['path'], list_arguments.archive, diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index 60f7f23c..a72701e5 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -23,9 +23,11 @@ def run_mount( repository, mount_arguments.repository ): if mount_arguments.archive: - logger.info(f'{repository["path"]}: Mounting archive {mount_arguments.archive}') + logger.info( + f'{repository.get("label", repository["path"])}: Mounting archive {mount_arguments.archive}' + ) else: # pragma: nocover - logger.info(f'{repository["path"]}: Mounting repository') + logger.info(f'{repository.get("label", repository["path"])}: Mounting repository') borgmatic.borg.mount.mount_archive( repository['path'], diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 2e25264b..422a9d46 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -37,7 +37,7 @@ def run_prune( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository["path"]}: Pruning archives{dry_run_label}') + logger.info(f'{repository.get("label", repository["path"])}: Pruning archives{dry_run_label}') borgmatic.borg.prune.prune_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index a3015c61..1bfc489b 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -23,7 +23,7 @@ def run_rcreate( ): return - logger.info(f'{repository["path"]}: Creating repository') + logger.info(f'{repository.get("label", repository["path"])}: Creating repository') borgmatic.borg.rcreate.create_repository( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 246c11a6..ded83f4f 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -73,12 +73,14 @@ def restore_single_database( Given (among other things) an archive name, a database hook name, and a configured database configuration dict, restore that database from the archive. ''' - logger.info(f'{repository}: Restoring database {database["name"]}') + logger.info( + f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}' + ) dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, database['name'], @@ -87,7 +89,7 @@ def restore_single_database( # Kick off a single database extract to stdout. extract_process = borgmatic.borg.extract.extract_archive( dry_run=global_arguments.dry_run, - repository=repository, + repository=repository['path'], archive=archive_name, paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]), location_config=location, @@ -106,7 +108,7 @@ def restore_single_database( borgmatic.hooks.dispatch.call_hooks( 'restore_database_dump', {hook_name: [database]}, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, @@ -265,7 +267,7 @@ def run_restore( return logger.info( - f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}' + f'{repository.get("label", repository["path"])}: Restoring databases from archive {restore_arguments.archive}' ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( @@ -314,7 +316,7 @@ def run_restore( found_names.add(database_name) restore_single_database( - repository['path'], + repository, location, storage, hooks, @@ -343,7 +345,7 @@ def run_restore( database['name'] = database_name restore_single_database( - repository['path'], + repository, location, storage, hooks, diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 279cd0e7..7756efd0 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -25,7 +25,9 @@ def run_rinfo( repository, rinfo_arguments.repository ): if not rinfo_arguments.json: # pragma: nocover - logger.answer(f'{repository["path"]}: Displaying repository summary information') + logger.answer( + f'{repository.get("label", repository["path"])}: Displaying repository summary information' + ) json_output = borgmatic.borg.rinfo.display_repository_info( repository['path'], diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index 50c59b6f..a9dee21d 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -25,7 +25,7 @@ def run_rlist( repository, rlist_arguments.repository ): if not rlist_arguments.json: # pragma: nocover - logger.answer(f'{repository["path"]}: Listing repository') + logger.answer(f'{repository.get("label", repository["path"])}: Listing repository') json_output = borgmatic.borg.rlist.list_repository( repository['path'], diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 36ac166d..df481e4d 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,7 +17,9 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' - logger.info(f'{repository["path"]}: Transferring archives to repository') + logger.info( + f'{repository.get("label", repository["path"])}: Transferring archives to repository' + ) borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 44396cd4..965f3931 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -113,10 +113,14 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository, retry_num = repo_queue.get() - logger.debug(f'{repository["path"]}: Running actions for repository') + logger.debug( + f'{repository.get("label", repository["path"])}: Running actions for repository' + ) timeout = retry_num * retry_wait if timeout: - logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') + logger.warning( + f'{repository.get("label", repository["path"])}: Sleeping {timeout}s before next retry' + ) time.sleep(timeout) try: yield from run_actions( @@ -139,14 +143,14 @@ def run_configuration(config_filename, config, arguments): ) tuple( # Consume the generator so as to trigger logging. log_error_records( - f'{repository["path"]}: Error running actions for repository', + f'{repository.get("label", repository["path"])}: Error running actions for repository', error, levelno=logging.WARNING, log_command_error_output=True, ) ) logger.warning( - f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}' + f'{repository.get("label", repository["path"])}: Retrying... attempt {retry_num + 1}/{retries}' ) continue @@ -154,7 +158,8 @@ def run_configuration(config_filename, config, arguments): return yield from log_error_records( - f'{repository["path"]}: Error running actions for repository', error + f'{repository.get("label", repository["path"])}: Error running actions for repository', + error, ) encountered_error = error error_repository = repository['path'] From 79b094d035d34844012347982b6c8aa5c2e070c8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 16 May 2023 09:59:09 -0700 Subject: [PATCH 196/344] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index bcfe22b2..21aec8a0 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.13.dev0 +1.7.13 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas diff --git a/setup.py b/setup.py index ce6b78e5..283089a4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.13.dev0' +VERSION = '1.7.13' setup( From e3425f48beca920f9845dfb31f644c8cd7c42367 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 16 May 2023 10:20:52 -0700 Subject: [PATCH 197/344] Instead of taking the first check time found, take the maximum value (#688) --- borgmatic/borg/check.py | 6 +++--- tests/unit/borg/test_check.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 63acbe26..930c82b6 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -299,7 +299,7 @@ def probe_for_check_time(location_config, borg_repository_id, check, archives_ch ~/.borgmatic/checks/1234567890/archives/9876543210 ~/.borgmatic/checks/1234567890/archives/all - ... and returns the modification time of the first file found (if any). The first path + ... and returns the maximum modification time of the files found (if any). The first path represents a more specific archives check time (a check on a subset of archives), and the second is a fallback to the last "all" archives check. @@ -318,8 +318,8 @@ def probe_for_check_time(location_config, borg_repository_id, check, archives_ch ) try: - return next(check_time for check_time in check_times if check_time) - except StopIteration: + return max(check_time for check_time in check_times if check_time) + except ValueError: return None diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index aad973bd..89db5d20 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -462,13 +462,13 @@ def test_read_check_time_on_missing_file_does_not_raise(): assert module.read_check_time('/path') is None -def test_probe_for_check_time_uses_first_of_multiple_check_times(): +def test_probe_for_check_time_uses_maximum_of_multiple_check_times(): flexmock(module).should_receive('make_check_time_path').and_return( '~/.borgmatic/checks/1234/archives/5678' ).and_return('~/.borgmatic/checks/1234/archives/all') flexmock(module).should_receive('read_check_time').and_return(1).and_return(2) - assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2 def test_probe_for_check_time_deduplicates_identical_check_time_paths(): From ee32b076eb666cefcee652cf8572f39b386649b8 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 16 May 2023 23:17:35 +0530 Subject: [PATCH 198/344] update tests and formatting --- borgmatic/actions/create.py | 18 +++--- tests/unit/actions/test_create.py | 6 +- tests/unit/borg/test_create.py | 94 +++++++++++++++---------------- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 2dba94ba..c5a6513d 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -1,15 +1,17 @@ import json -import os import logging +import os -import importlib_metadata +try: + import importlib_metadata +except ModuleNotFoundError: # pragma: nocover + import importlib.metadata as importlib_metadata import borgmatic.borg.create import borgmatic.config.validate import borgmatic.hooks.command import borgmatic.hooks.dispatch import borgmatic.hooks.dump - from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY logger = logging.getLogger(__name__) @@ -23,10 +25,8 @@ def create_borgmatic_manifest(location, config_paths, dry_run): if dry_run: return - borgmatic_source_directory = ( - location.get('borgmatic_source_directory') - if location.get('borgmatic_source_directory') - else DEFAULT_BORGMATIC_SOURCE_DIRECTORY + borgmatic_source_directory = location.get( + 'borgmatic_source_directory', DEFAULT_BORGMATIC_SOURCE_DIRECTORY ) borgmatic_manifest_path = os.path.expanduser( @@ -36,13 +36,13 @@ def create_borgmatic_manifest(location, config_paths, dry_run): if not os.path.exists(borgmatic_manifest_path): os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True) - with open(borgmatic_manifest_path, 'w') as f: + with open(borgmatic_manifest_path, 'w') as config_list_file: json.dump( { 'borgmatic_version': importlib_metadata.version('borgmatic'), 'config_paths': config_paths, }, - f, + config_list_file, ) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 2b724085..f6d9acba 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -19,7 +19,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): json=flexmock(), list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) list( module.run_create( @@ -52,7 +52,7 @@ def test_run_create_runs_with_selected_repository(): json=flexmock(), list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) list( module.run_create( @@ -85,7 +85,7 @@ def test_run_create_bails_if_repository_does_not_match(): json=flexmock(), list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) list( module.run_create( diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index e0462e2d..efabd0ff 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -492,7 +492,7 @@ def test_create_archive_calls_borg_with_parameters(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -536,7 +536,7 @@ def test_create_archive_calls_borg_with_environment(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -582,7 +582,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -628,7 +628,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -672,7 +672,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -713,7 +713,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, ) @@ -758,7 +758,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -799,7 +799,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, ) @@ -843,7 +843,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -889,7 +889,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stats=True, ) @@ -933,7 +933,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte }, storage_config={'checkpoint_interval': 600}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -976,7 +976,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume }, storage_config={'checkpoint_volume': 1024}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1019,7 +1019,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param }, storage_config={'chunker_params': '1,2,3,4'}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1062,7 +1062,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( }, storage_config={'compression': 'rle'}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1111,7 +1111,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ }, storage_config={'upload_rate_limit': 100}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1157,7 +1157,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1201,7 +1201,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1251,7 +1251,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1305,7 +1305,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1361,7 +1361,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1416,7 +1416,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1471,7 +1471,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1515,7 +1515,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1558,7 +1558,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), local_path='borg1', ) @@ -1602,7 +1602,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), remote_path='borg1', ) @@ -1646,7 +1646,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): }, storage_config={'umask': 740}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1689,7 +1689,7 @@ def test_create_archive_with_log_json_calls_borg_with_log_json_parameters(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=True), + global_arguments=flexmock(log_json=True, used_config_paths=[]), ) @@ -1732,7 +1732,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): }, storage_config={'lock_wait': 5}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1775,7 +1775,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stats=True, ) @@ -1819,7 +1819,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), list_files=True, ) @@ -1869,7 +1869,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), progress=True, ) @@ -1913,7 +1913,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), progress=True, ) @@ -1974,7 +1974,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), progress=True, stream_processes=processes, ) @@ -2039,7 +2039,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, ) @@ -2107,7 +2107,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, ) @@ -2172,7 +2172,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, ) @@ -2213,7 +2213,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, ) @@ -2256,7 +2256,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, stats=True, ) @@ -2304,7 +2304,7 @@ def test_create_archive_with_source_directories_glob_expands(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2348,7 +2348,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2391,7 +2391,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2434,7 +2434,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): }, storage_config={'archive_name_format': 'ARCHIVE_NAME'}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2478,7 +2478,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2522,7 +2522,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2565,7 +2565,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): }, storage_config={'extra_borg_options': {'create': '--extra --options'}}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2626,7 +2626,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, ) @@ -2652,7 +2652,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) From 833796d1c466eb89deca796cf2d4371cc1e389d8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 17 May 2023 08:48:54 -0700 Subject: [PATCH 199/344] Add archive check probing logic tweak to NEWS (#688). --- NEWS | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 21aec8a0..9308507a 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.7.14.dev0 + * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist. + 1.7.13 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" flag. See the documentation for more information: diff --git a/setup.py b/setup.py index 283089a4..2665af85 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.13' +VERSION = '1.7.14.dev0' setup( From 05b989347cda9d4939e61fd7409e789136e89cd3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 23 May 2023 08:43:45 -0700 Subject: [PATCH 200/344] Upgrade requests test requirement (security). --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 6516a500..56160dc8 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -25,7 +25,7 @@ jsonschema==4.17.3 pytest==7.3.0 pytest-cov==4.0.0 regex; python_version >= '3.8' -requests==2.28.2 +requests==2.31.0 ruamel.yaml>0.15.0,<0.18.0 toml==0.10.2; python_version >= '3.8' typed-ast; python_version >= '3.8' From 9299841a5b22028899057f167e8f4e7cd67cc7e1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 23 May 2023 14:30:16 -0700 Subject: [PATCH 201/344] Add date-based matching flags to NEWS (#659). --- NEWS | 1 + tests/integration/borg/test_commands.py | 42 ++++++++++++------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/NEWS b/NEWS index 9308507a..85bfa38e 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.7.14.dev0 * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist. + * #659: Add Borg 2 date-based matching flags to various actions for archive selection. 1.7.13 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 3347890c..11fbe981 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -68,6 +68,27 @@ def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): ) +def test_prune_archives_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('prune')['prune'] + flexmock(borgmatic.borg.prune).should_receive('execute_command').replace_with( + assert_command_does_not_duplicate_flags + ) + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.prune.prune_archives( + False, + 'repo', + {}, + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + argparse.Namespace(log_json=False), + ) + + def test_make_list_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments('list')['list'] @@ -124,24 +145,3 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): fuzz_argument(arguments, argument_name), argparse.Namespace(log_json=False), ) - - -def test_prune_archives_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('prune')['prune'] - flexmock(borgmatic.borg.prune).should_receive('execute_command').replace_with( - assert_command_does_not_duplicate_flags - ) - - for argument_name in dir(arguments): - if argument_name.startswith('_'): - continue - - borgmatic.borg.prune.prune_archives( - False, - 'repo', - {}, - {}, - '2.3.4', - fuzz_argument(arguments, argument_name), - argparse.Namespace(log_json=False), - ) From 76138faaf385a27563dc5271c03eef3e90c83a97 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 23 May 2023 14:49:04 -0700 Subject: [PATCH 202/344] Add integration test for mount action (#659). --- tests/integration/borg/test_commands.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 11fbe981..a76d43f1 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -5,6 +5,8 @@ from flexmock import flexmock import borgmatic.borg.info import borgmatic.borg.list +import borgmatic.borg.mount +import borgmatic.borg.prune import borgmatic.borg.rlist import borgmatic.borg.transfer import borgmatic.commands.arguments @@ -89,6 +91,28 @@ def test_prune_archives_command_does_not_duplicate_flags_or_raise(): ) +def test_mount_archive_command_does_not_duplicate_flags_or_raise(): + arguments = borgmatic.commands.arguments.parse_arguments('mount', '--mount-point', 'tmp')[ + 'mount' + ] + flexmock(borgmatic.borg.mount).should_receive('execute_command').replace_with( + assert_command_does_not_duplicate_flags + ) + + for argument_name in dir(arguments): + if argument_name.startswith('_'): + continue + + borgmatic.borg.mount.mount_archive( + 'repo', + 'archive', + fuzz_argument(arguments, argument_name), + {}, + '2.3.4', + argparse.Namespace(log_json=False), + ) + + def test_make_list_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments('list')['list'] From 84c21b062f024d8498f6c45ed06e3dbc73f6e8e1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 23 May 2023 16:55:40 -0700 Subject: [PATCH 203/344] Fix incorrect argument ordering (#659). --- borgmatic/actions/prune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index f7280840..deaea384 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -44,8 +44,8 @@ def run_prune( storage, retention, local_borg_version, - global_arguments, prune_arguments, + global_arguments, local_path=local_path, remote_path=remote_path, ) From 2241de11c00c8f4dc138ce89750fb5470912301d Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 26 May 2023 00:26:13 +0530 Subject: [PATCH 204/344] start work on borgmatic config bootstrap command --- borgmatic/commands/arguments.py | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 0812edea..832c62d9 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -9,6 +9,8 @@ SUBPARSER_ALIASES = { 'compact': [], 'create': ['-C'], 'check': ['-k'], + 'config': [], + 'config_bootstrap': [], 'extract': ['-x'], 'export-tar': [], 'mount': ['-m'], @@ -523,6 +525,66 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + config_parser = subparsers.add_parser( + 'config', + aliases=SUBPARSER_ALIASES['config'], + help='Perform configuration file related operations', + description='Perform configuration file related operations', + add_help=False, + parents=[top_level_parser], + ) + + config_subparsers = config_parser.add_subparsers( + title='config subcommands', + description='Valid subcommands for config', + help='Additional help', + ) + + config_bootstrap_parser = config_subparsers.add_parser( + 'bootstrap', + aliases=SUBPARSER_ALIASES['config_bootstrap'], + help='Extract files from a borgmatic created repository to the current directory', + description='Extract a named archive from a borgmatic created repository to the current directory without a configuration file', + add_help=False, + parents=[config_parser], + ) + config_bootstrap_group = config_bootstrap_parser.add_argument_group('config bootstrap arguments') + config_bootstrap_group.add_argument( + '--repository', + help='Path of repository to extract', + required=True, + ) + config_bootstrap_group.add_argument( + '--archive', help='Name of archive to extract, defaults to "latest"' + ) + config_bootstrap_group.add_argument( + '--path', + '--restore-path', + metavar='PATH', + nargs='+', + dest='paths', + help='Paths to extract from archive, defaults to the entire archive', + ) + config_bootstrap_group.add_argument( + '--destination', + metavar='PATH', + dest='destination', + help='Directory to extract files into, defaults to the current directory', + ) + config_bootstrap_group.add_argument( + '--strip-components', + type=lambda number: number if number == 'all' else int(number), + metavar='NUMBER', + help='Number of leading path components to remove from each extracted path or "all" to strip all leading path components. Skip paths with fewer elements', + ) + config_bootstrap_group.add_argument( + '--progress', + dest='progress', + default=False, + action='store_true', + help='Display progress for each file as it is extracted', + ) + export_tar_parser = subparsers.add_parser( 'export-tar', aliases=SUBPARSER_ALIASES['export-tar'], From 8b7996dfda6656a0324e1d920d85a6c7922c47ec Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 26 May 2023 01:07:11 +0530 Subject: [PATCH 205/344] removed parents and used reversed remaining_args --- borgmatic/commands/arguments.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 832c62d9..4f9ca967 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -48,7 +48,7 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): if 'borg' in unparsed_arguments: subparsers = {'borg': subparsers['borg']} - for argument in remaining_arguments: + for argument in reversed(remaining_arguments): canonical_name = alias_to_subparser_name.get(argument, argument) subparser = subparsers.get(canonical_name) @@ -531,7 +531,11 @@ def make_parsers(): help='Perform configuration file related operations', description='Perform configuration file related operations', add_help=False, - parents=[top_level_parser], + ) + + config_group = config_parser.add_argument_group('config arguments') + config_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' ) config_subparsers = config_parser.add_subparsers( @@ -546,7 +550,6 @@ def make_parsers(): help='Extract files from a borgmatic created repository to the current directory', description='Extract a named archive from a borgmatic created repository to the current directory without a configuration file', add_help=False, - parents=[config_parser], ) config_bootstrap_group = config_bootstrap_parser.add_argument_group('config bootstrap arguments') config_bootstrap_group.add_argument( @@ -584,6 +587,9 @@ def make_parsers(): action='store_true', help='Display progress for each file as it is extracted', ) + config_bootstrap_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) export_tar_parser = subparsers.add_parser( 'export-tar', From 96adee444bd26184ee993c1b3f5474ccbf857dea Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 25 May 2023 15:03:15 -0700 Subject: [PATCH 206/344] Potential fix for nested subparsers not parsing correctly. --- borgmatic/commands/arguments.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 4f9ca967..e0ffb47e 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -48,7 +48,7 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): if 'borg' in unparsed_arguments: subparsers = {'borg': subparsers['borg']} - for argument in reversed(remaining_arguments): + for argument in remaining_arguments: canonical_name = alias_to_subparser_name.get(argument, argument) subparser = subparsers.get(canonical_name) @@ -58,7 +58,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): # If a parsed value happens to be the same as the name of a subparser, remove it from the # remaining arguments. This prevents, for instance, "check --only extract" from triggering # the "extract" subparser. - parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments) + parsed, unused_remaining = subparser.parse_known_args( + [argument for argument in unparsed_arguments if argument != canonical_name] + ) for value in vars(parsed).values(): if isinstance(value, str): if value in subparsers: @@ -85,7 +87,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): continue subparser = subparsers[subparser_name] - unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments) + unused_parsed, remaining_arguments = subparser.parse_known_args( + [argument for argument in remaining_arguments if argument != subparser_name] + ) # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the # "borg" action. From f512d1e460aa0bc807e6b92dd4b2ffaafd283d2f Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Mon, 1 May 2023 03:31:45 +0530 Subject: [PATCH 207/344] add verbosity level -2 Signed-off-by: Soumik Dutta --- borgmatic/commands/arguments.py | 16 ++++++++-------- borgmatic/logger.py | 8 ++++++-- borgmatic/verbosity.py | 2 ++ sample/systemd/borgmatic.service | 2 +- tests/unit/test_verbosity.py | 2 ++ 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 07d1f6d7..0880c473 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -152,30 +152,30 @@ def make_parsers(): '-v', '--verbosity', type=int, - choices=range(-1, 3), + choices=range(-2, 3), default=0, - help='Display verbose progress to the console (from only errors to very verbose: -1, 0, 1, or 2)', + help='Display verbose progress to the console (from disabled, errors to very verbose: -2, -1, 0, 1, or 2)', ) global_group.add_argument( '--syslog-verbosity', type=int, - choices=range(-1, 3), + choices=range(-2, 3), default=0, - help='Log verbose progress to syslog (from only errors to very verbose: -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given', + help='Log verbose progress to syslog (from disabled, errors to very verbose: -2, -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given', ) global_group.add_argument( '--log-file-verbosity', type=int, - choices=range(-1, 3), + choices=range(-2, 3), default=0, - help='Log verbose progress to log file (from only errors to very verbose: -1, 0, 1, or 2). Only used when --log-file is given', + help='Log verbose progress to log file (from disabled, errors to very verbose: -2, -1, 0, 1, or 2). Only used when --log-file is given', ) global_group.add_argument( '--monitoring-verbosity', type=int, - choices=range(-1, 3), + choices=range(-2, 3), default=0, - help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)', + help='Log verbose progress to monitoring integrations that support logging (from disabled, errors to very verbose: -2, -1, 0, 1, or 2)', ) global_group.add_argument( '--log-file', diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 52065928..4cb3674e 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -141,6 +141,7 @@ def add_logging_level(level_name, level_number): ANSWER = logging.WARN - 5 +DISABLED = logging.DEBUG - 5 def add_custom_log_levels(): # pragma: no cover @@ -148,6 +149,7 @@ def add_custom_log_levels(): # pragma: no cover Add a custom log level between WARN and INFO for user-requested answers. ''' add_logging_level('ANSWER', ANSWER) + add_logging_level('DISABLED', DISABLED) def configure_logging( @@ -175,10 +177,12 @@ def configure_logging( # Log certain log levels to console stderr and others to stdout. This supports use cases like # grepping (non-error) output. + console_disabled = logging.NullHandler() console_error_handler = logging.StreamHandler(sys.stderr) console_standard_handler = logging.StreamHandler(sys.stdout) console_handler = Multi_stream_handler( { + logging.DISABLED: console_disabled, logging.CRITICAL: console_error_handler, logging.ERROR: console_error_handler, logging.WARN: console_error_handler, @@ -191,7 +195,7 @@ def configure_logging( console_handler.setLevel(console_log_level) syslog_path = None - if log_file is None: + if log_file is None and syslog_log_level != logging.DISABLED: if os.path.exists('/dev/log'): syslog_path = '/dev/log' elif os.path.exists('/var/run/syslog'): @@ -206,7 +210,7 @@ def configure_logging( ) syslog_handler.setLevel(syslog_log_level) handlers = (console_handler, syslog_handler) - elif log_file: + elif log_file and log_file_log_level != logging.DISABLED: file_handler = logging.handlers.WatchedFileHandler(log_file) file_handler.setFormatter( logging.Formatter( diff --git a/borgmatic/verbosity.py b/borgmatic/verbosity.py index cdadd61f..8cba009d 100644 --- a/borgmatic/verbosity.py +++ b/borgmatic/verbosity.py @@ -2,6 +2,7 @@ import logging import borgmatic.logger +VERBOSITY_DISABLED = -2 VERBOSITY_ERROR = -1 VERBOSITY_ANSWER = 0 VERBOSITY_SOME = 1 @@ -15,6 +16,7 @@ def verbosity_to_log_level(verbosity): borgmatic.logger.add_custom_log_levels() return { + VERBOSITY_DISABLED: logging.DISABLED, VERBOSITY_ERROR: logging.ERROR, VERBOSITY_ANSWER: logging.ANSWER, VERBOSITY_SOME: logging.INFO, diff --git a/sample/systemd/borgmatic.service b/sample/systemd/borgmatic.service index 3b2dff65..885c435f 100644 --- a/sample/systemd/borgmatic.service +++ b/sample/systemd/borgmatic.service @@ -61,4 +61,4 @@ LogRateLimitIntervalSec=0 # Delay start to prevent backups running during boot. Note that systemd-inhibit requires dbus and # dbus-user-session to be installed. ExecStartPre=sleep 1m -ExecStart=systemd-inhibit --who="borgmatic" --what="sleep:shutdown" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --verbosity -1 --syslog-verbosity 1 +ExecStart=systemd-inhibit --who="borgmatic" --what="sleep:shutdown" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --verbosity -2 --syslog-verbosity 1 diff --git a/tests/unit/test_verbosity.py b/tests/unit/test_verbosity.py index 572ad192..a53e4edb 100644 --- a/tests/unit/test_verbosity.py +++ b/tests/unit/test_verbosity.py @@ -17,11 +17,13 @@ def insert_logging_mock(log_level): def test_verbosity_to_log_level_maps_known_verbosity_to_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.logging).DISABLED = module.borgmatic.logger.DISABLED assert module.verbosity_to_log_level(module.VERBOSITY_ERROR) == logging.ERROR assert module.verbosity_to_log_level(module.VERBOSITY_ANSWER) == module.borgmatic.logger.ANSWER assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO assert module.verbosity_to_log_level(module.VERBOSITY_LOTS) == logging.DEBUG + assert module.verbosity_to_log_level(module.VERBOSITY_DISABLED) == logging.DISABLED def test_verbosity_to_log_level_maps_unknown_verbosity_to_warning_level(): From 99473c30a8d9c5fa09201589121518e08923a0f7 Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Wed, 3 May 2023 19:01:07 +0530 Subject: [PATCH 208/344] disable sending logs in Healthchecks Signed-off-by: Soumik Dutta --- borgmatic/hooks/healthchecks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index 4cafc49f..af2bcd67 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -76,7 +76,7 @@ def initialize_monitor(hook_config, config_filename, monitoring_log_level, dry_r we can send them all to Healthchecks upon a finish or failure state. But skip this if the "send_logs" option is false. ''' - if hook_config.get('send_logs') is False: + if hook_config.get('send_logs') is False or monitoring_log_level == logging.DISABLED: return ping_body_limit = max( From 6a2eb1f1579ba3c640eab0cb80abb51f9bb52d48 Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Wed, 3 May 2023 20:10:19 +0530 Subject: [PATCH 209/344] make value of disabled level higher so that no other log has higher priority Signed-off-by: Soumik Dutta --- borgmatic/hooks/healthchecks.py | 2 +- borgmatic/logger.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index af2bcd67..4cafc49f 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -76,7 +76,7 @@ def initialize_monitor(hook_config, config_filename, monitoring_log_level, dry_r we can send them all to Healthchecks upon a finish or failure state. But skip this if the "send_logs" option is false. ''' - if hook_config.get('send_logs') is False or monitoring_log_level == logging.DISABLED: + if hook_config.get('send_logs') is False: return ping_body_limit = max( diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 4cb3674e..cda5151d 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -141,7 +141,7 @@ def add_logging_level(level_name, level_number): ANSWER = logging.WARN - 5 -DISABLED = logging.DEBUG - 5 +DISABLED = logging.CRITICAL + 10 def add_custom_log_levels(): # pragma: no cover From 5a9bb4b97fdc4a2c5c808743205222b73d73c150 Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Mon, 8 May 2023 17:54:04 +0530 Subject: [PATCH 210/344] update help strings Signed-off-by: Soumik Dutta --- borgmatic/commands/arguments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 0880c473..8f478a29 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -154,14 +154,14 @@ def make_parsers(): type=int, choices=range(-2, 3), default=0, - help='Display verbose progress to the console (from disabled, errors to very verbose: -2, -1, 0, 1, or 2)', + help='Display verbose progress to the console (from disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2)', ) global_group.add_argument( '--syslog-verbosity', type=int, choices=range(-2, 3), default=0, - help='Log verbose progress to syslog (from disabled, errors to very verbose: -2, -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given', + help='Log verbose progress to syslog (from disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given', ) global_group.add_argument( '--log-file-verbosity', @@ -175,7 +175,7 @@ def make_parsers(): type=int, choices=range(-2, 3), default=0, - help='Log verbose progress to monitoring integrations that support logging (from disabled, errors to very verbose: -2, -1, 0, 1, or 2)', + help='Log verbose progress to monitoring integrations that support logging (from disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2)', ) global_group.add_argument( '--log-file', From 24be6272ed3a5ce78ec4befea0cf1dd3b39eb31a Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Tue, 9 May 2023 20:32:34 +0530 Subject: [PATCH 211/344] add test for logger Signed-off-by: Soumik Dutta --- tests/unit/test_logger.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index dc9c748d..6d3ce240 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -284,6 +284,23 @@ def test_configure_logging_skips_syslog_if_interactive_console(): module.configure_logging(console_log_level=logging.INFO) +def test_configure_logging_skips_syslog_if_syslog_logging_is_disabled(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.logging).DISABLED = module.DISABLED + flexmock(module).should_receive('Multi_stream_handler').and_return( + flexmock(setFormatter=lambda formatter: None, setLevel=lambda level: None) + ) + flexmock(module).should_receive('Console_color_formatter') + flexmock(module).should_receive('syslog_path').and_return(None) + flexmock(module).should_receive('interactive_console').never() + flexmock(module.logging).should_receive('basicConfig').with_args( + level=logging.DISABLED, handlers=tuple + ) + flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True) + flexmock(module.logging.handlers).should_receive('SysLogHandler').never() + + module.configure_logging(console_log_level=logging.INFO, syslog_log_level=logging.DISABLED) + def test_configure_logging_to_log_file_instead_of_syslog(): flexmock(module).should_receive('add_custom_log_levels') From 09f59ad97dc83ea761118e15de212c7e9f1c29d1 Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Fri, 12 May 2023 19:05:52 +0530 Subject: [PATCH 212/344] disable monitoring hooks if monitoring_log_level is set to DISABLED Signed-off-by: Soumik Dutta --- borgmatic/commands/borgmatic.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 965f3931..597868f6 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -36,7 +36,7 @@ from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments from borgmatic.config import checks, collect, convert, validate from borgmatic.hooks import command, dispatch, monitor -from borgmatic.logger import add_custom_log_levels, configure_logging, should_do_markup +from borgmatic.logger import add_custom_log_levels, configure_logging, should_do_markup, DISABLED from borgmatic.signals import configure_signals from borgmatic.verbosity import verbosity_to_log_level @@ -70,6 +70,7 @@ def run_configuration(config_filename, config, arguments): error_repository = '' using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) + monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED try: local_borg_version = borg_version.local_borg_version(storage, local_path) @@ -78,7 +79,7 @@ def run_configuration(config_filename, config, arguments): return try: - if using_primary_action: + if monitoring_hooks_are_activated: dispatch.call_hooks( 'initialize_monitor', hooks, @@ -87,7 +88,7 @@ def run_configuration(config_filename, config, arguments): monitoring_log_level, global_arguments.dry_run, ) - if using_primary_action: + dispatch.call_hooks( 'ping_monitor', hooks, @@ -165,7 +166,7 @@ def run_configuration(config_filename, config, arguments): error_repository = repository['path'] try: - if using_primary_action: + if monitoring_hooks_are_activated: # send logs irrespective of error dispatch.call_hooks( 'ping_monitor', @@ -185,7 +186,7 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: try: - if using_primary_action: + if monitoring_hooks_are_activated: dispatch.call_hooks( 'ping_monitor', hooks, From f98d07e8d8adb67d14a1e97e66f61fe4615690ed Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Fri, 12 May 2023 19:13:03 +0530 Subject: [PATCH 213/344] fix logger test Signed-off-by: Soumik Dutta --- tests/unit/test_logger.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 6d3ce240..5dca649a 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -291,10 +291,9 @@ def test_configure_logging_skips_syslog_if_syslog_logging_is_disabled(): flexmock(setFormatter=lambda formatter: None, setLevel=lambda level: None) ) flexmock(module).should_receive('Console_color_formatter') - flexmock(module).should_receive('syslog_path').and_return(None) flexmock(module).should_receive('interactive_console').never() flexmock(module.logging).should_receive('basicConfig').with_args( - level=logging.DISABLED, handlers=tuple + level=logging.INFO, handlers=tuple ) flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True) flexmock(module.logging.handlers).should_receive('SysLogHandler').never() From d556a23f97be17c7658c21632c7e9bb7595b7589 Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Fri, 12 May 2023 20:00:11 +0530 Subject: [PATCH 214/344] update borgmatic tests Signed-off-by: Soumik Dutta --- tests/unit/commands/test_borgmatic.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index bd98c01f..144de18a 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -169,6 +169,20 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): assert results == [] +def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_are_disabled(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(module.DISABLED) + flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) + + flexmock(module.dispatch).should_receive('call_hooks').never() + flexmock(module).should_receive('run_actions').and_return([]) + flexmock(module.dispatch).should_receive('call_hooks').never() + + config = {'location': {'repositories': [{'path': 'foo'}]}} + arguments = {'global': flexmock(monitoring_verbosity=-2, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == [] + + def test_run_configuration_logs_on_error_hook_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) @@ -229,8 +243,7 @@ def test_run_configuration_retries_hard_error(): ).and_return([flexmock()]) error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', - OSError, + 'foo: Error running actions for repository', OSError, ).and_return(error_logs) config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} From 0283f9ae2af55b1e82e373faab4b9a4f1d2f0a1f Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Fri, 12 May 2023 20:02:38 +0530 Subject: [PATCH 215/344] fix help string Signed-off-by: Soumik Dutta --- borgmatic/commands/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8f478a29..f5a2c408 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -168,7 +168,7 @@ def make_parsers(): type=int, choices=range(-2, 3), default=0, - help='Log verbose progress to log file (from disabled, errors to very verbose: -2, -1, 0, 1, or 2). Only used when --log-file is given', + help='Log verbose progress to log file (from disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Only used when --log-file is given', ) global_group.add_argument( '--monitoring-verbosity', From 3d41ed3a34bd782677d25a375f093ad48ff4279d Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Fri, 26 May 2023 09:58:53 +0530 Subject: [PATCH 216/344] add test to check that log_file is disabled if logging is disabled Signed-off-by: Soumik Dutta --- tests/unit/test_logger.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 5dca649a..84072de3 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -284,6 +284,7 @@ def test_configure_logging_skips_syslog_if_interactive_console(): module.configure_logging(console_log_level=logging.INFO) + def test_configure_logging_skips_syslog_if_syslog_logging_is_disabled(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).DISABLED = module.DISABLED @@ -301,6 +302,27 @@ def test_configure_logging_skips_syslog_if_syslog_logging_is_disabled(): module.configure_logging(console_log_level=logging.INFO, syslog_log_level=logging.DISABLED) +def test_configure_logging_skips_log_file_if_log_file_logging_is_disabled(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.logging).DISABLED = module.DISABLED + flexmock(module).should_receive('Multi_stream_handler').and_return( + flexmock(setFormatter=lambda formatter: None, setLevel=lambda level: None) + ) + + # syslog skipped in non-interactive console if --log-file argument provided + flexmock(module).should_receive('interactive_console').and_return(False) + flexmock(module.logging).should_receive('basicConfig').with_args( + level=logging.INFO, handlers=tuple + ) + flexmock(module.os.path).should_receive('exists').with_args('/dev/log').never() + flexmock(module.logging.handlers).should_receive('SysLogHandler').never() + flexmock(module.logging.handlers).should_receive('WatchedFileHandler').never() + + module.configure_logging( + console_log_level=logging.INFO, log_file_log_level=logging.DISABLED, log_file='/tmp/logfile' + ) + + def test_configure_logging_to_log_file_instead_of_syslog(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER From dbb778a4d6ed472b6a39d861f43e6a61502c799d Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 26 May 2023 22:44:31 +0530 Subject: [PATCH 217/344] finish parsing and add error for empty config subcommand --- borgmatic/commands/arguments.py | 30 ++++++++++++++++++++++++------ borgmatic/commands/borgmatic.py | 3 ++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e0ffb47e..5678da13 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -42,6 +42,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): for subparser_name, aliases in SUBPARSER_ALIASES.items() for alias in aliases } + subcommand_parsers_mapping = { + 'config': ['bootstrap'], + } # If the "borg" action is used, skip all other subparsers. This avoids confusion like # "borg list" triggering borgmatic's own list action. @@ -70,7 +73,18 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): if item in subparsers: remaining_arguments.remove(item) - arguments[canonical_name] = parsed + if canonical_name not in subcommand_parsers_mapping: + arguments[canonical_name] = parsed + else: + arguments[canonical_name] = None + + for argument in arguments: + if arguments[argument] == None: + for subcommand in subcommand_parsers_mapping[argument]: + if subcommand not in arguments: + raise ValueError("Missing subcommand for {}. Expected one of {}".format( + argument, subcommand_parsers_mapping[argument] + )) # If no actions are explicitly requested, assume defaults. if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments: @@ -81,8 +95,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): remaining_arguments = list(unparsed_arguments) - # Now ask each subparser, one by one, to greedily consume arguments. - for subparser_name, subparser in subparsers.items(): + # Now ask each subparser, one by one, to greedily consume arguments, from last to first. This + # allows subparsers to consume arguments before their parent subparsers do. + for subparser_name, subparser in reversed(subparsers.items()): if subparser_name not in arguments.keys(): continue @@ -937,7 +952,7 @@ def make_parsers(): ) borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - return top_level_parser, subparsers + return top_level_parser, subparsers, config_subparsers def parse_arguments(*unparsed_arguments): @@ -945,10 +960,13 @@ def parse_arguments(*unparsed_arguments): Given command-line arguments with which this script was invoked, parse the arguments and return them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance. ''' - top_level_parser, subparsers = make_parsers() + top_level_parser, subparsers, config_subparsers = make_parsers() + + subparser_choices = subparsers.choices.copy() + subparser_choices.update(config_subparsers.choices) arguments, remaining_arguments = parse_subparser_arguments( - unparsed_arguments, subparsers.choices + unparsed_arguments, subparser_choices ) arguments['global'] = top_level_parser.parse_args(remaining_arguments) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 1eb68349..4271fa8f 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -616,7 +616,8 @@ def collect_configuration_run_summary_logs(configs, arguments): if 'extract' in arguments or 'mount' in arguments: validate.guard_single_repository_selected(repository, configs) - validate.guard_configuration_contains_repository(repository, configs) + if 'config' not in arguments: + validate.guard_configuration_contains_repository(repository, configs) except ValueError as error: yield from log_error_records(str(error)) return From 4c60bf84d71941eecf406c22e6adc79684daebd4 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sun, 28 May 2023 01:36:32 +0530 Subject: [PATCH 218/344] extract config files --- borgmatic/actions/bootstrap.py | 73 +++++++++++++++++++++++++++++++++ borgmatic/commands/borgmatic.py | 14 +++++++ 2 files changed, 87 insertions(+) create mode 100644 borgmatic/actions/bootstrap.py diff --git a/borgmatic/actions/bootstrap.py b/borgmatic/actions/bootstrap.py new file mode 100644 index 00000000..d51cafc6 --- /dev/null +++ b/borgmatic/actions/bootstrap.py @@ -0,0 +1,73 @@ +import logging +import os +import json + +import borgmatic.borg.extract +import borgmatic.borg.rlist +import borgmatic.config.validate +import borgmatic.hooks.command + +from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY + +logger = logging.getLogger(__name__) + +def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): + borgmatic_source_directory = DEFAULT_BORGMATIC_SOURCE_DIRECTORY + borgmatic_manifest_path = os.path.expanduser( + os.path.join(borgmatic_source_directory, 'bootstrap', 'configs-list.json') + ) + extract_process = borgmatic.borg.extract.extract_archive( + global_arguments.dry_run, + bootstrap_arguments.repository, + borgmatic.borg.rlist.resolve_archive_name( + bootstrap_arguments.repository, + bootstrap_arguments.archive or 'latest', + {}, + local_borg_version, + global_arguments + ), + [borgmatic_manifest_path], + {}, + {}, + local_borg_version, + global_arguments, + extract_to_stdout=True, + ) + + manifest_data = json.loads(extract_process.stdout.read()) + + return manifest_data['config_paths'] + + + + +def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): + ''' + Run the "bootstrap" action for the given repository. + ''' + manifest_config_paths = get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) + + for config_path in manifest_config_paths: + logger.info('Bootstrapping config path %s', config_path) + + borgmatic.borg.extract.extract_archive( + global_arguments.dry_run, + bootstrap_arguments.repository, + borgmatic.borg.rlist.resolve_archive_name( + bootstrap_arguments.repository, + bootstrap_arguments.archive or 'latest', + {}, + local_borg_version, + global_arguments + ), + [config_path], + {}, + {}, + local_borg_version, + global_arguments, + extract_to_stdout=False, + ) + + + + diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 4271fa8f..ccc6ccf2 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -21,6 +21,7 @@ import borgmatic.actions.compact import borgmatic.actions.create import borgmatic.actions.export_tar import borgmatic.actions.extract +import borgmatic.actions.bootstrap import borgmatic.actions.info import borgmatic.actions.list import borgmatic.actions.mount @@ -621,6 +622,19 @@ def collect_configuration_run_summary_logs(configs, arguments): except ValueError as error: yield from log_error_records(str(error)) return + + if 'bootstrap' in arguments: + # no configuration file is needed for bootstrap + local_borg_version = borg_version.local_borg_version({}, 'borg') + borgmatic.actions.bootstrap.run_bootstrap(arguments['bootstrap'], arguments['global'], local_borg_version) + yield logging.makeLogRecord( + dict( + levelno=logging.INFO, + levelname='INFO', + msg='Bootstrap successful', + ) + ) + return if not configs: yield from log_error_records( From a7f81d538de55d3fed40e1181e06f270626a53c6 Mon Sep 17 00:00:00 2001 From: Soumik Dutta Date: Mon, 29 May 2023 01:09:00 +0530 Subject: [PATCH 219/344] nit changes - help strings in borgmatic commands - test fixes in test_logger and test_borgmatic Signed-off-by: Soumik Dutta --- borgmatic/commands/arguments.py | 6 +++--- tests/unit/commands/test_borgmatic.py | 1 - tests/unit/test_logger.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index f5a2c408..6039e4fa 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -154,21 +154,21 @@ def make_parsers(): type=int, choices=range(-2, 3), default=0, - help='Display verbose progress to the console (from disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2)', + help='Display verbose progress to the console (disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2)', ) global_group.add_argument( '--syslog-verbosity', type=int, choices=range(-2, 3), default=0, - help='Log verbose progress to syslog (from disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given', + help='Log verbose progress to syslog (disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given', ) global_group.add_argument( '--log-file-verbosity', type=int, choices=range(-2, 3), default=0, - help='Log verbose progress to log file (from disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Only used when --log-file is given', + help='Log verbose progress to log file (disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Only used when --log-file is given', ) global_group.add_argument( '--monitoring-verbosity', diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 144de18a..9b8bf582 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -175,7 +175,6 @@ def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_ar flexmock(module.dispatch).should_receive('call_hooks').never() flexmock(module).should_receive('run_actions').and_return([]) - flexmock(module.dispatch).should_receive('call_hooks').never() config = {'location': {'repositories': [{'path': 'foo'}]}} arguments = {'global': flexmock(monitoring_verbosity=-2, dry_run=False), 'create': flexmock()} diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 84072de3..f86aacd7 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -314,7 +314,7 @@ def test_configure_logging_skips_log_file_if_log_file_logging_is_disabled(): flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.INFO, handlers=tuple ) - flexmock(module.os.path).should_receive('exists').with_args('/dev/log').never() + flexmock(module.os.path).should_receive('exists').never() flexmock(module.logging.handlers).should_receive('SysLogHandler').never() flexmock(module.logging.handlers).should_receive('WatchedFileHandler').never() @@ -335,7 +335,7 @@ def test_configure_logging_to_log_file_instead_of_syslog(): flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.DEBUG, handlers=tuple ) - flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True) + flexmock(module.os.path).should_receive('exists').never() flexmock(module.logging.handlers).should_receive('SysLogHandler').never() file_handler = logging.handlers.WatchedFileHandler('/tmp/logfile') flexmock(module.logging.handlers).should_receive('WatchedFileHandler').with_args( From b222f6a60b810406c722a02d92cb963407ad1ed0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 30 May 2023 15:52:49 -0700 Subject: [PATCH 220/344] Mention new verbosity level to NEWS (#484). --- NEWS | 2 ++ tests/unit/commands/test_borgmatic.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 85bfa38e..f4e08d84 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.7.14.dev0 + * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file, + or monitoring), so not even errors are shown. * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist. * #659: Add Borg 2 date-based matching flags to various actions for archive selection. diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 9b8bf582..4ca802d5 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -242,7 +242,8 @@ def test_run_configuration_retries_hard_error(): ).and_return([flexmock()]) error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', OSError, + 'foo: Error running actions for repository', + OSError, ).and_return(error_logs) config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} From 341bd4118d7707a5bb80571a285fefbe741c0eca Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 30 May 2023 16:53:55 -0700 Subject: [PATCH 221/344] Fix "--archive latest" on "list" and "info" actions only working on the first of multiple configured repositories (#706). --- NEWS | 2 ++ borgmatic/actions/arguments.py | 9 +++++++++ borgmatic/actions/info.py | 5 +++-- borgmatic/actions/list.py | 6 ++++-- tests/unit/actions/test_arguments.py | 10 ++++++++++ tests/unit/actions/test_info.py | 3 +++ tests/unit/actions/test_list.py | 3 +++ 7 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 borgmatic/actions/arguments.py create mode 100644 tests/unit/actions/test_arguments.py diff --git a/NEWS b/NEWS index f4e08d84..b4a92eab 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,8 @@ or monitoring), so not even errors are shown. * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist. * #659: Add Borg 2 date-based matching flags to various actions for archive selection. + * #706: Fix "--archive latest" on "list" and "info" actions only working on the first of multiple + configured repositories. 1.7.13 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" diff --git a/borgmatic/actions/arguments.py b/borgmatic/actions/arguments.py new file mode 100644 index 00000000..7fd77fc8 --- /dev/null +++ b/borgmatic/actions/arguments.py @@ -0,0 +1,9 @@ +import argparse + + +def update_arguments(arguments, **updates): + ''' + Given an argparse.Namespace instance of command-line arguments and one or more keyword argument + updates to perform, return a copy of the arguments with those updates applied. + ''' + return argparse.Namespace(**dict(vars(arguments), **updates)) diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index 91699623..0116fd70 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -1,6 +1,7 @@ import json import logging +import borgmatic.actions.arguments import borgmatic.borg.info import borgmatic.borg.rlist import borgmatic.config.validate @@ -29,7 +30,7 @@ def run_info( logger.answer( f'{repository.get("label", repository["path"])}: Displaying archive summary information' ) - info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( + archive_name = borgmatic.borg.rlist.resolve_archive_name( repository['path'], info_arguments.archive, storage, @@ -42,7 +43,7 @@ def run_info( repository['path'], storage, local_borg_version, - info_arguments, + borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name), global_arguments, local_path, remote_path, diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 720fab1c..667062d8 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -1,6 +1,7 @@ import json import logging +import borgmatic.actions.arguments import borgmatic.borg.list import borgmatic.config.validate @@ -29,7 +30,8 @@ def run_list( logger.answer(f'{repository.get("label", repository["path"])}: Searching archives') elif not list_arguments.archive: logger.answer(f'{repository.get("label", repository["path"])}: Listing archives') - list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( + + archive_name = borgmatic.borg.rlist.resolve_archive_name( repository['path'], list_arguments.archive, storage, @@ -42,7 +44,7 @@ def run_list( repository['path'], storage, local_borg_version, - list_arguments, + borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name), global_arguments, local_path, remote_path, diff --git a/tests/unit/actions/test_arguments.py b/tests/unit/actions/test_arguments.py new file mode 100644 index 00000000..988cbede --- /dev/null +++ b/tests/unit/actions/test_arguments.py @@ -0,0 +1,10 @@ +from borgmatic.actions import arguments as module + + +def test_update_arguments_copies_and_updates_without_modifying_original(): + original = module.argparse.Namespace(foo=1, bar=2, baz=3) + + result = module.update_arguments(original, bar=7, baz=8) + + assert original == module.argparse.Namespace(foo=1, bar=2, baz=3) + assert result == module.argparse.Namespace(foo=1, bar=7, baz=8) diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py index 97161968..748d866f 100644 --- a/tests/unit/actions/test_info.py +++ b/tests/unit/actions/test_info.py @@ -9,6 +9,9 @@ def test_run_info_does_not_raise(): flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( flexmock() ) + flexmock(module.borgmatic.actions.arguments).should_receive('update_arguments').and_return( + flexmock() + ) flexmock(module.borgmatic.borg.info).should_receive('display_archives_info') info_arguments = flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()) diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py index 5ee72251..07a1a58c 100644 --- a/tests/unit/actions/test_list.py +++ b/tests/unit/actions/test_list.py @@ -9,6 +9,9 @@ def test_run_list_does_not_raise(): flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( flexmock() ) + flexmock(module.borgmatic.actions.arguments).should_receive('update_arguments').and_return( + flexmock() + ) flexmock(module.borgmatic.borg.list).should_receive('list_archive') list_arguments = flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()) From 518aeabb2a860378049b769354b8187ea86f4e64 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 30 May 2023 22:25:27 -0700 Subject: [PATCH 222/344] Document verbosity levels (#484). --- docs/how-to/inspect-your-backups.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index e89d23f6..c1e509b8 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -24,6 +24,15 @@ Or, for even more progress and debug spew: borgmatic --verbosity 2 ``` +The full set of verbosity levels are: + + * `-2`: disable output entirely New in borgmatic 1.7.14 + * `-1`: only show errors + * `0`: default output + * `1`: some additional output (informational level) + * `2`: lots of additional output (debug level) + + ## Backup summary If you're less concerned with progress during a backup, and you only want to From 8f4cce5fa5eba86a519f463ca20f95ef2c301d4f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 30 May 2023 22:30:06 -0700 Subject: [PATCH 223/344] Make dev docs message stand out a little more. --- docs/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docker-compose.yaml b/docs/docker-compose.yaml index e854c13a..dfadf4a4 100644 --- a/docs/docker-compose.yaml +++ b/docs/docker-compose.yaml @@ -17,6 +17,6 @@ services: - sh - -c - | - echo "You can view dev docs at http://localhost:8080" + echo; echo "You can view dev docs at http://localhost:8080"; echo depends_on: - docs From 1784ca59104ec462d1921303040ab0463204022c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 30 May 2023 23:19:33 -0700 Subject: [PATCH 224/344] Fix "check" action error when repository and archive checks are configured but the archive check gets skipped due to the configured frequency (#704). --- NEWS | 2 ++ borgmatic/borg/check.py | 2 +- borgmatic/commands/borgmatic.py | 2 +- tests/unit/borg/test_check.py | 8 +++++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index b4a92eab..da8a3d68 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,8 @@ or monitoring), so not even errors are shown. * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist. * #659: Add Borg 2 date-based matching flags to various actions for archive selection. + * #704: Fix "check" action error when repository and archive checks are configured but the archive + check gets skipped due to the configured frequency. * #706: Fix "--archive latest" on "list" and "info" actions only working on the first of multiple configured repositories. diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 930c82b6..0c417aca 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -226,7 +226,7 @@ def make_check_flags(checks, archive_filter_flags): else: data_flags = () - common_flags = archive_filter_flags + data_flags + common_flags = (archive_filter_flags if 'archives' in checks else ()) + data_flags if {'repository', 'archives'}.issubset(set(checks)): return common_flags diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 597868f6..535c048e 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -36,7 +36,7 @@ from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments from borgmatic.config import checks, collect, convert, validate from borgmatic.hooks import command, dispatch, monitor -from borgmatic.logger import add_custom_log_levels, configure_logging, should_do_markup, DISABLED +from borgmatic.logger import DISABLED, add_custom_log_levels, configure_logging, should_do_markup from borgmatic.signals import configure_signals from borgmatic.verbosity import verbosity_to_log_level diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 89db5d20..a1044ba4 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -356,12 +356,18 @@ def test_make_check_flags_with_archives_check_returns_flag(): assert flags == ('--archives-only',) -def test_make_check_flags_with_archive_filtler_flags_includes_those_flags(): +def test_make_check_flags_with_archives_check_and_archive_filter_flags_includes_those_flags(): flags = module.make_check_flags(('archives',), ('--match-archives', 'sh:foo-*')) assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') +def test_make_check_flags_without_archives_check_and_with_archive_filter_flags_includes_those_flags(): + flags = module.make_check_flags(('repository',), ('--match-archives', 'sh:foo-*')) + + assert flags == ('--repository-only',) + + def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) From 4f49b345af5a475752518620609de7620c5903cf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 30 May 2023 23:21:55 -0700 Subject: [PATCH 225/344] NEWS wording fix for clarity (#706). --- NEWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index da8a3d68..ed1a9aa9 100644 --- a/NEWS +++ b/NEWS @@ -5,8 +5,8 @@ * #659: Add Borg 2 date-based matching flags to various actions for archive selection. * #704: Fix "check" action error when repository and archive checks are configured but the archive check gets skipped due to the configured frequency. - * #706: Fix "--archive latest" on "list" and "info" actions only working on the first of multiple - configured repositories. + * #706: Fix "--archive latest" on "list" and "info" actions that only worked on the first of + multiple configured repositories. 1.7.13 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" From 74aa28e02797adc1aab3003e2012f8b125fbea3e Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 1 Jun 2023 16:53:34 +0530 Subject: [PATCH 226/344] support more flags --- borgmatic/actions/bootstrap.py | 11 +++++++++-- borgmatic/commands/arguments.py | 12 ++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/borgmatic/actions/bootstrap.py b/borgmatic/actions/bootstrap.py index d51cafc6..46ba53f5 100644 --- a/borgmatic/actions/bootstrap.py +++ b/borgmatic/actions/bootstrap.py @@ -12,7 +12,7 @@ from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY logger = logging.getLogger(__name__) def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): - borgmatic_source_directory = DEFAULT_BORGMATIC_SOURCE_DIRECTORY + borgmatic_source_directory = bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY borgmatic_manifest_path = os.path.expanduser( os.path.join(borgmatic_source_directory, 'bootstrap', 'configs-list.json') ) @@ -34,7 +34,11 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): extract_to_stdout=True, ) - manifest_data = json.loads(extract_process.stdout.read()) + try: + manifest_data = json.loads(extract_process.stdout.read()) + except json.decoder.JSONDecodeError as error: + logger.error('Error parsing manifest data: %s', error) + raise return manifest_data['config_paths'] @@ -66,6 +70,9 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): local_borg_version, global_arguments, extract_to_stdout=False, + destination_path=bootstrap_arguments.destination, + strip_components=bootstrap_arguments.strip_components, + progress=bootstrap_arguments.progress, ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 5678da13..3ad87252 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -577,21 +577,17 @@ def make_parsers(): required=True, ) config_bootstrap_group.add_argument( - '--archive', help='Name of archive to extract, defaults to "latest"' + '--borgmatic-source-directory', + help='Path of the borgmatic source directory if other than the default', ) config_bootstrap_group.add_argument( - '--path', - '--restore-path', - metavar='PATH', - nargs='+', - dest='paths', - help='Paths to extract from archive, defaults to the entire archive', + '--archive', help='Name of archive to extract, defaults to "latest"' ) config_bootstrap_group.add_argument( '--destination', metavar='PATH', dest='destination', - help='Directory to extract files into, defaults to the current directory', + help='Directory to extract files into, defaults to /', ) config_bootstrap_group.add_argument( '--strip-components', From bb60b25399c89def1bfe289a019ec83a8cc08155 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 2 Jun 2023 02:04:35 +0530 Subject: [PATCH 227/344] merge subparsers and refactor --- borgmatic/actions/bootstrap.py | 2 +- borgmatic/commands/arguments.py | 42 +++++++++++++++++++++------------ borgmatic/commands/borgmatic.py | 19 ++++++++------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/borgmatic/actions/bootstrap.py b/borgmatic/actions/bootstrap.py index 46ba53f5..3a480dd1 100644 --- a/borgmatic/actions/bootstrap.py +++ b/borgmatic/actions/bootstrap.py @@ -70,7 +70,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): local_borg_version, global_arguments, extract_to_stdout=False, - destination_path=bootstrap_arguments.destination, + destination_path=bootstrap_arguments.destination or '/', strip_components=bootstrap_arguments.strip_components, progress=bootstrap_arguments.progress, ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 3ad87252..a66f0edc 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,3 +1,4 @@ +import argparse import collections from argparse import Action, ArgumentParser @@ -73,18 +74,15 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): if item in subparsers: remaining_arguments.remove(item) - if canonical_name not in subcommand_parsers_mapping: - arguments[canonical_name] = parsed - else: - arguments[canonical_name] = None + arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed for argument in arguments: - if arguments[argument] == None: - for subcommand in subcommand_parsers_mapping[argument]: - if subcommand not in arguments: - raise ValueError("Missing subcommand for {}. Expected one of {}".format( - argument, subcommand_parsers_mapping[argument] - )) + if not arguments[argument]: + if not any(subcommand in arguments for subcommand in subcommand_parsers_mapping[argument]): + raise ValueError("Missing subcommand for {}. Expected one of {}".format( + argument, subcommand_parsers_mapping[argument] + )) + # If no actions are explicitly requested, assume defaults. if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments: @@ -948,7 +946,17 @@ def make_parsers(): ) borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - return top_level_parser, subparsers, config_subparsers + merged_subparsers = argparse._SubParsersAction(None, None, metavar=None, dest='merged', parser_class=None) + + for name, subparser in subparsers.choices.items(): + merged_subparsers._name_parser_map[name] = subparser + subparser._name_parser_map = merged_subparsers._name_parser_map + + for name, subparser in config_subparsers.choices.items(): + merged_subparsers._name_parser_map[name] = subparser + subparser._name_parser_map = merged_subparsers._name_parser_map + + return top_level_parser, merged_subparsers def parse_arguments(*unparsed_arguments): @@ -956,14 +964,18 @@ def parse_arguments(*unparsed_arguments): Given command-line arguments with which this script was invoked, parse the arguments and return them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance. ''' - top_level_parser, subparsers, config_subparsers = make_parsers() + top_level_parser, subparsers = make_parsers() - subparser_choices = subparsers.choices.copy() - subparser_choices.update(config_subparsers.choices) arguments, remaining_arguments = parse_subparser_arguments( - unparsed_arguments, subparser_choices + unparsed_arguments, subparsers.choices ) + + if 'bootstrap' in arguments.keys() and len(arguments.keys()) > 1: + raise ValueError( + 'The bootstrap action cannot be combined with other actions. Please run it separately.' + ) + arguments['global'] = top_level_parser.parse_args(remaining_arguments) if arguments['global'].excludes_filename: diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index ccc6ccf2..7aaf5708 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -617,7 +617,7 @@ def collect_configuration_run_summary_logs(configs, arguments): if 'extract' in arguments or 'mount' in arguments: validate.guard_single_repository_selected(repository, configs) - if 'config' not in arguments: + if 'bootstrap' not in arguments: validate.guard_configuration_contains_repository(repository, configs) except ValueError as error: yield from log_error_records(str(error)) @@ -626,14 +626,17 @@ def collect_configuration_run_summary_logs(configs, arguments): if 'bootstrap' in arguments: # no configuration file is needed for bootstrap local_borg_version = borg_version.local_borg_version({}, 'borg') - borgmatic.actions.bootstrap.run_bootstrap(arguments['bootstrap'], arguments['global'], local_borg_version) - yield logging.makeLogRecord( - dict( - levelno=logging.INFO, - levelname='INFO', - msg='Bootstrap successful', + try: + borgmatic.actions.bootstrap.run_bootstrap(arguments['bootstrap'], arguments['global'], local_borg_version) + yield logging.makeLogRecord( + dict( + levelno=logging.INFO, + levelname='INFO', + msg='Bootstrap successful', + ) ) - ) + except (CalledProcessError, ValueError, OSError) as error: + yield from log_error_records('Error running bootstrap', error) return if not configs: From caf654366caa4267196bc49ab0e56d526d7b7ef0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 3 Jun 2023 10:19:34 -0700 Subject: [PATCH 228/344] Document work-around for colons in YAML strings (#708). --- borgmatic/config/schema.yaml | 30 +++++++++---------- ...reparation-and-cleanup-steps-to-backups.md | 10 +++++++ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 903c0432..78db2902 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -605,7 +605,7 @@ properties: List of one or more shell commands or scripts to execute before all the actions for each repository. example: - - echo "Starting actions." + - "echo Starting actions." before_backup: type: array items: @@ -614,7 +614,7 @@ properties: List of one or more shell commands or scripts to execute before creating a backup, run once per repository. example: - - echo "Starting a backup." + - "echo Starting a backup." before_prune: type: array items: @@ -623,7 +623,7 @@ properties: List of one or more shell commands or scripts to execute before pruning, run once per repository. example: - - echo "Starting pruning." + - "echo Starting pruning." before_compact: type: array items: @@ -632,7 +632,7 @@ properties: List of one or more shell commands or scripts to execute before compaction, run once per repository. example: - - echo "Starting compaction." + - "echo Starting compaction." before_check: type: array items: @@ -641,7 +641,7 @@ properties: List of one or more shell commands or scripts to execute before consistency checks, run once per repository. example: - - echo "Starting checks." + - "echo Starting checks." before_extract: type: array items: @@ -650,7 +650,7 @@ properties: List of one or more shell commands or scripts to execute before extracting a backup, run once per repository. example: - - echo "Starting extracting." + - "echo Starting extracting." after_backup: type: array items: @@ -659,7 +659,7 @@ properties: List of one or more shell commands or scripts to execute after creating a backup, run once per repository. example: - - echo "Finished a backup." + - "echo Finished a backup." after_compact: type: array items: @@ -668,7 +668,7 @@ properties: List of one or more shell commands or scripts to execute after compaction, run once per repository. example: - - echo "Finished compaction." + - "echo Finished compaction." after_prune: type: array items: @@ -677,7 +677,7 @@ properties: List of one or more shell commands or scripts to execute after pruning, run once per repository. example: - - echo "Finished pruning." + - "echo Finished pruning." after_check: type: array items: @@ -686,7 +686,7 @@ properties: List of one or more shell commands or scripts to execute after consistency checks, run once per repository. example: - - echo "Finished checks." + - "echo Finished checks." after_extract: type: array items: @@ -695,7 +695,7 @@ properties: List of one or more shell commands or scripts to execute after extracting a backup, run once per repository. example: - - echo "Finished extracting." + - "echo Finished extracting." after_actions: type: array items: @@ -704,7 +704,7 @@ properties: List of one or more shell commands or scripts to execute after all actions for each repository. example: - - echo "Finished actions." + - "echo Finished actions." on_error: type: array items: @@ -715,7 +715,7 @@ properties: "compact", or "check" action or an associated before/after hook. example: - - echo "Error during create/prune/compact/check." + - "echo Error during create/prune/compact/check." before_everything: type: array items: @@ -726,7 +726,7 @@ properties: These are collected from all configuration files and then run once before all of them (prior to all actions). example: - - echo "Starting actions." + - "echo Starting actions." after_everything: type: array items: @@ -737,7 +737,7 @@ properties: These are collected from all configuration files and then run once after all of them (after any action). example: - - echo "Completed actions." + - "echo Completed actions." postgresql_databases: type: array items: diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 426e1a85..eeba2e04 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -28,6 +28,16 @@ hooks: - umount /some/filesystem ``` +If your command contains a special YAML character such as a colon, you may +need to quote the entire string (or use a [multiline +string](https://yaml-multiline.info/)) to avoid an error: + +```yaml +hooks: + before_backup: + - "echo Backup: start" +``` + New in version 1.6.0 The `before_backup` and `after_backup` hooks each run once per repository in a configuration file. `before_backup` hooks runs right before the `create` From ce6daff12f624dfa44132115aa3b519f1e3ab748 Mon Sep 17 00:00:00 2001 From: Felix Kaechele Date: Tue, 23 May 2023 17:18:46 -0400 Subject: [PATCH 229/344] Fix importlib.metadata.files workaround Some distributions, such as Fedora, do not install the RECORDS file as part of a package's dist-info. As a result importlib.metadata.files will return None. Use the workaround for these cases as well. Signed-off-by: Felix Kaechele --- borgmatic/config/validate.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index b39199fe..1c9b1050 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -16,19 +16,19 @@ def schema_filename(): ''' Path to the installed YAML configuration schema file, used to validate and parse the configuration. - - Raise FileNotFoundError when the schema path does not exist. ''' - try: - return next( - str(path.locate()) - for path in importlib_metadata.files('borgmatic') - if path.match('config/schema.yaml') - ) - except StopIteration: - # If the schema wasn't found in the package's files, this is probably a pip editable - # install, so try a different approach to get the schema. - return os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml') + + files = importlib_metadata.files('borgmatic') + if files is not None: + try: + return next(str(path.locate()) for path in files if path.match('config/schema.yaml')) + except StopIteration: + # schema not found in package, fall through to the approach below + pass + + # If the schema wasn't found in the package's files, this is probably a pip editable + # install, so try a different approach to get the schema. + return os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml') def format_json_error_path_element(path_element): From 15cabb93cab55c3f61a30c9e6bb0fdbf6d06b304 Mon Sep 17 00:00:00 2001 From: Felix Kaechele Date: Fri, 2 Jun 2023 21:35:33 -0400 Subject: [PATCH 230/344] Drop importlib_metadata entirely The fallback option using the dirname of the config module location seems to be more robust in a number of cases. Signed-off-by: Felix Kaechele --- borgmatic/config/validate.py | 16 ---------------- tests/unit/config/test_validate.py | 19 +------------------ 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 1c9b1050..2ed1af5d 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -3,11 +3,6 @@ import os import jsonschema import ruamel.yaml -try: - import importlib_metadata -except ModuleNotFoundError: # pragma: nocover - import importlib.metadata as importlib_metadata - import borgmatic.config from borgmatic.config import environment, load, normalize, override @@ -17,17 +12,6 @@ def schema_filename(): Path to the installed YAML configuration schema file, used to validate and parse the configuration. ''' - - files = importlib_metadata.files('borgmatic') - if files is not None: - try: - return next(str(path.locate()) for path in files if path.match('config/schema.yaml')) - except StopIteration: - # schema not found in package, fall through to the approach below - pass - - # If the schema wasn't found in the package's files, this is probably a pip editable - # install, so try a different approach to get the schema. return os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml') diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index e81f2b02..fdb13b68 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -5,24 +5,7 @@ from borgmatic.config import validate as module def test_schema_filename_finds_schema_path(): - schema_path = '/var/borgmatic/config/schema.yaml' - - flexmock(module.importlib_metadata).should_receive('files').and_return( - flexmock(match=lambda path: False, locate=lambda: None), - flexmock(match=lambda path: True, locate=lambda: schema_path), - flexmock(match=lambda path: False, locate=lambda: None), - ) - - assert module.schema_filename() == schema_path - - -def test_schema_filename_with_missing_schema_path_in_package_still_finds_it_in_config_directory(): - flexmock(module.importlib_metadata).should_receive('files').and_return( - flexmock(match=lambda path: False, locate=lambda: None), - flexmock(match=lambda path: False, locate=lambda: None), - ) - - assert module.schema_filename().endswith('/borgmatic/config/schema.yaml') + module.schema_filename().endswith('/borgmatic/config/schema.yaml') def test_format_json_error_path_element_formats_array_index(): From ba0899660dd3e1bfd0ee6264c7001885719efc42 Mon Sep 17 00:00:00 2001 From: Felix Kaechele Date: Sat, 3 Jun 2023 08:11:56 -0400 Subject: [PATCH 231/344] Verify that schema path exists before returning it Signed-off-by: Felix Kaechele --- borgmatic/config/validate.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 2ed1af5d..1917eea0 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -11,8 +11,15 @@ def schema_filename(): ''' Path to the installed YAML configuration schema file, used to validate and parse the configuration. + + Raise FileNotFoundError when the schema path does not exist. ''' - return os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml') + schema_path = os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml') + + if os.path.exists(schema_path) and os.path.isfile(schema_path): + return schema_path + + raise FileNotFoundError def format_json_error_path_element(path_element): From c61d63b235f6e2bcd59a88baea85012704453983 Mon Sep 17 00:00:00 2001 From: Felix Kaechele Date: Sun, 4 Jun 2023 00:50:35 -0400 Subject: [PATCH 232/344] Use open() to test for file existance and readability Signed-off-by: Felix Kaechele --- borgmatic/config/validate.py | 4 +--- tests/unit/config/test_validate.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 1917eea0..5835ead1 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -16,11 +16,9 @@ def schema_filename(): ''' schema_path = os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml') - if os.path.exists(schema_path) and os.path.isfile(schema_path): + with open(schema_path): return schema_path - raise FileNotFoundError - def format_json_error_path_element(path_element): ''' diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index fdb13b68..701ba510 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -1,3 +1,7 @@ +import os +import sys +from io import StringIO + import pytest from flexmock import flexmock @@ -5,7 +9,23 @@ from borgmatic.config import validate as module def test_schema_filename_finds_schema_path(): - module.schema_filename().endswith('/borgmatic/config/schema.yaml') + schema_path = '/var/borgmatic/config/schema.yaml' + + flexmock(os.path).should_receive('dirname').and_return("/var/borgmatic/config") + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args(schema_path).and_return(StringIO()) + assert module.schema_filename() == schema_path + + +def test_schema_filename_raises_filenotfounderror(): + schema_path = '/var/borgmatic/config/schema.yaml' + + flexmock(os.path).should_receive('dirname').and_return("/var/borgmatic/config") + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args(schema_path).and_raise(FileNotFoundError) + + with pytest.raises(FileNotFoundError): + module.schema_filename() def test_format_json_error_path_element_formats_array_index(): From 1a5b3c9e4e911697df79ca25749a5d5a313f0f50 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 3 Jun 2023 22:07:24 -0700 Subject: [PATCH 233/344] Add Fedora schema loading fix to NEWS (#703). --- NEWS | 1 + tests/unit/config/test_validate.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index ed1a9aa9..f33743fc 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ or monitoring), so not even errors are shown. * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist. * #659: Add Borg 2 date-based matching flags to various actions for archive selection. + * #703: Fix an error when loading the configuration schema on Fedora Linux. * #704: Fix "check" action error when repository and archive checks are configured but the archive check gets skipped due to the configured frequency. * #706: Fix "--archive latest" on "list" and "info" actions that only worked on the first of diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 701ba510..4ab2b762 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -11,7 +11,7 @@ from borgmatic.config import validate as module def test_schema_filename_finds_schema_path(): schema_path = '/var/borgmatic/config/schema.yaml' - flexmock(os.path).should_receive('dirname').and_return("/var/borgmatic/config") + flexmock(os.path).should_receive('dirname').and_return('/var/borgmatic/config') builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args(schema_path).and_return(StringIO()) assert module.schema_filename() == schema_path @@ -20,7 +20,7 @@ def test_schema_filename_finds_schema_path(): def test_schema_filename_raises_filenotfounderror(): schema_path = '/var/borgmatic/config/schema.yaml' - flexmock(os.path).should_receive('dirname').and_return("/var/borgmatic/config") + flexmock(os.path).should_receive('dirname').and_return('/var/borgmatic/config') builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args(schema_path).and_raise(FileNotFoundError) From e80f27f9221d0e7e628708f4e230d21a1d2d0608 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 3 Jun 2023 22:14:21 -0700 Subject: [PATCH 234/344] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index f33743fc..87252c7b 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.14.dev0 +1.7.14 * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file, or monitoring), so not even errors are shown. * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist. diff --git a/setup.py b/setup.py index 2665af85..df0718de 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.14.dev0' +VERSION = '1.7.14' setup( From a185eb73b0f51ca50679c49410a6b310d50e52cb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 3 Jun 2023 22:26:49 -0700 Subject: [PATCH 235/344] Fix GitHub release script now that "master" has been renamed to "main". --- scripts/release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release b/scripts/release index 22e20175..c7a4d5d5 100755 --- a/scripts/release +++ b/scripts/release @@ -50,5 +50,5 @@ curl --silent --request POST \ --header "Content-Type: application/json" \ --data "{\"body\": \"$escaped_release_changelog\", \"draft\": false, \"name\": \"borgmatic $version\", \"prerelease\": false, \"tag_name\": \"$version\"}" -github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" \ +github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" --target_commit="main" \ --name="borgmatic $version" --body="$release_changelog" From b5d9398910c95a7bbde1486aa33d64d0210136a1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 3 Jun 2023 22:37:46 -0700 Subject: [PATCH 236/344] Stop uploading GPG signatures to pypi since it no longer supports them. --- scripts/release | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/release b/scripts/release index c7a4d5d5..b4ab65cf 100755 --- a/scripts/release +++ b/scripts/release @@ -35,10 +35,8 @@ git push github $version rm -fr dist python3 setup.py bdist_wheel python3 setup.py sdist -gpg --detach-sign --armor dist/borgmatic-*.tar.gz -gpg --detach-sign --armor dist/borgmatic-*-py3-none-any.whl -twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz dist/borgmatic-*.tar.gz.asc -twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none-any.whl.asc +twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz +twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl # Set release changelogs on projects.torsion.org and GitHub. release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')" From a6425b8867e80e4b2bc4ce31e0dedeb461337666 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 4 Jun 2023 22:21:16 -0700 Subject: [PATCH 237/344] Fix moved Arch Linux borgmatic URL. --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 515f017e..08aea148 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -89,7 +89,7 @@ installing borgmatic: * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic) * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic) - * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) + * [Arch Linux](https://archlinux.org/packages/extra/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) * [OpenBSD](https://openports.pl/path/sysutils/borgmatic) * [openSUSE](https://software.opensuse.org/package/borgmatic) From 206a9c960743c21bc69b43e2b54eacba8e611221 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 5 Jun 2023 20:05:10 +0530 Subject: [PATCH 238/344] edit schema comments and work on witten review --- borgmatic/actions/{ => config}/bootstrap.py | 14 +++++------- borgmatic/actions/create.py | 2 +- borgmatic/commands/arguments.py | 25 +++++++++++---------- borgmatic/commands/borgmatic.py | 6 ++--- 4 files changed, 22 insertions(+), 25 deletions(-) rename borgmatic/actions/{ => config}/bootstrap.py (81%) diff --git a/borgmatic/actions/bootstrap.py b/borgmatic/actions/config/bootstrap.py similarity index 81% rename from borgmatic/actions/bootstrap.py rename to borgmatic/actions/config/bootstrap.py index 3a480dd1..f3b64594 100644 --- a/borgmatic/actions/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -14,14 +14,14 @@ logger = logging.getLogger(__name__) def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): borgmatic_source_directory = bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY borgmatic_manifest_path = os.path.expanduser( - os.path.join(borgmatic_source_directory, 'bootstrap', 'configs-list.json') + os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json') ) extract_process = borgmatic.borg.extract.extract_archive( global_arguments.dry_run, bootstrap_arguments.repository, borgmatic.borg.rlist.resolve_archive_name( bootstrap_arguments.repository, - bootstrap_arguments.archive or 'latest', + bootstrap_arguments.archive, {}, local_borg_version, global_arguments @@ -34,11 +34,7 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): extract_to_stdout=True, ) - try: - manifest_data = json.loads(extract_process.stdout.read()) - except json.decoder.JSONDecodeError as error: - logger.error('Error parsing manifest data: %s', error) - raise + manifest_data = json.loads(extract_process.stdout.read()) return manifest_data['config_paths'] @@ -59,7 +55,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): bootstrap_arguments.repository, borgmatic.borg.rlist.resolve_archive_name( bootstrap_arguments.repository, - bootstrap_arguments.archive or 'latest', + bootstrap_arguments.archive, {}, local_borg_version, global_arguments @@ -70,7 +66,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): local_borg_version, global_arguments, extract_to_stdout=False, - destination_path=bootstrap_arguments.destination or '/', + destination_path=bootstrap_arguments.destination, strip_components=bootstrap_arguments.strip_components, progress=bootstrap_arguments.progress, ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index c5a6513d..48f863e6 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -30,7 +30,7 @@ def create_borgmatic_manifest(location, config_paths, dry_run): ) borgmatic_manifest_path = os.path.expanduser( - os.path.join(borgmatic_source_directory, 'bootstrap', 'configs-list.json') + os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json') ) if not os.path.exists(borgmatic_manifest_path): diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index a66f0edc..cbfc9be6 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -74,15 +74,15 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): if item in subparsers: remaining_arguments.remove(item) - arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed - + try: + arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed + except UnboundLocalError: + pass + for argument in arguments: if not arguments[argument]: if not any(subcommand in arguments for subcommand in subcommand_parsers_mapping[argument]): - raise ValueError("Missing subcommand for {}. Expected one of {}".format( - argument, subcommand_parsers_mapping[argument] - )) - + raise ValueError(f"Missing subcommand for {argument}. Expected one of {subcommand_parsers_mapping[argument]}") # If no actions are explicitly requested, assume defaults. if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments: @@ -564,28 +564,29 @@ def make_parsers(): config_bootstrap_parser = config_subparsers.add_parser( 'bootstrap', aliases=SUBPARSER_ALIASES['config_bootstrap'], - help='Extract files from a borgmatic created repository to the current directory', - description='Extract a named archive from a borgmatic created repository to the current directory without a configuration file', + help='Extract the config files used to create a borgmatic repository', + description='Extract just the config files that were used to create a borgmatic repository during the "create" operation', add_help=False, ) config_bootstrap_group = config_bootstrap_parser.add_argument_group('config bootstrap arguments') config_bootstrap_group.add_argument( '--repository', - help='Path of repository to extract', + help='Path of repository to extract config files from', required=True, ) config_bootstrap_group.add_argument( '--borgmatic-source-directory', - help='Path of the borgmatic source directory if other than the default', + help='Path that stores the config files used to create an archive, and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic', ) config_bootstrap_group.add_argument( - '--archive', help='Name of archive to extract, defaults to "latest"' + '--archive', help='Name of archive to extract config files from, defaults to "latest"', default='latest' ) config_bootstrap_group.add_argument( '--destination', metavar='PATH', dest='destination', - help='Directory to extract files into, defaults to /', + help='Directory to extract config files into, defaults to /', + default='/', ) config_bootstrap_group.add_argument( '--strip-components', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 7aaf5708..ddfb9898 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -21,7 +21,7 @@ import borgmatic.actions.compact import borgmatic.actions.create import borgmatic.actions.export_tar import borgmatic.actions.extract -import borgmatic.actions.bootstrap +import borgmatic.actions.config.bootstrap import borgmatic.actions.info import borgmatic.actions.list import borgmatic.actions.mount @@ -627,7 +627,7 @@ def collect_configuration_run_summary_logs(configs, arguments): # no configuration file is needed for bootstrap local_borg_version = borg_version.local_borg_version({}, 'borg') try: - borgmatic.actions.bootstrap.run_bootstrap(arguments['bootstrap'], arguments['global'], local_borg_version) + borgmatic.actions.config.bootstrap.run_bootstrap(arguments['bootstrap'], arguments['global'], local_borg_version) yield logging.makeLogRecord( dict( levelno=logging.INFO, @@ -635,7 +635,7 @@ def collect_configuration_run_summary_logs(configs, arguments): msg='Bootstrap successful', ) ) - except (CalledProcessError, ValueError, OSError) as error: + except (CalledProcessError, ValueError, OSError, json.JSONDecodeError, KeyError) as error: yield from log_error_records('Error running bootstrap', error) return From 6a1d1a2e5902f36a43cebbda25f3ee4dac292fb0 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 5 Jun 2023 20:31:09 +0530 Subject: [PATCH 239/344] fix indentation error that caused too many test failures --- borgmatic/commands/arguments.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index cbfc9be6..a7667c31 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -74,10 +74,10 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): if item in subparsers: remaining_arguments.remove(item) - try: - arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed - except UnboundLocalError: - pass + try: + arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed + except UnboundLocalError: + pass for argument in arguments: if not arguments[argument]: @@ -972,7 +972,7 @@ def parse_arguments(*unparsed_arguments): unparsed_arguments, subparsers.choices ) - if 'bootstrap' in arguments.keys() and len(arguments.keys()) > 1: + if 'bootstrap' in arguments.keys() and 'config' in arguments.keys() and len(arguments.keys()) > 2: raise ValueError( 'The bootstrap action cannot be combined with other actions. Please run it separately.' ) From 4b024daae0bea31020bad0967cb3b0c2552a097b Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 6 Jun 2023 23:37:09 +0530 Subject: [PATCH 240/344] pass all tests with wittens recommendation --- borgmatic/actions/config/bootstrap.py | 22 +++++----- borgmatic/commands/arguments.py | 62 ++++++++++++++++++++------- borgmatic/commands/borgmatic.py | 8 ++-- 3 files changed, 62 insertions(+), 30 deletions(-) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index f3b64594..5b323451 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -1,18 +1,20 @@ +import json import logging import os -import json import borgmatic.borg.extract import borgmatic.borg.rlist import borgmatic.config.validate import borgmatic.hooks.command - from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY logger = logging.getLogger(__name__) + def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): - borgmatic_source_directory = bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY + borgmatic_source_directory = ( + bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY + ) borgmatic_manifest_path = os.path.expanduser( os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json') ) @@ -24,7 +26,7 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): bootstrap_arguments.archive, {}, local_borg_version, - global_arguments + global_arguments, ), [borgmatic_manifest_path], {}, @@ -38,14 +40,14 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): return manifest_data['config_paths'] - - def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): ''' Run the "bootstrap" action for the given repository. ''' - manifest_config_paths = get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) + manifest_config_paths = get_config_paths( + bootstrap_arguments, global_arguments, local_borg_version + ) for config_path in manifest_config_paths: logger.info('Bootstrapping config path %s', config_path) @@ -58,7 +60,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): bootstrap_arguments.archive, {}, local_borg_version, - global_arguments + global_arguments, ), [config_path], {}, @@ -70,7 +72,3 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): strip_components=bootstrap_arguments.strip_components, progress=bootstrap_arguments.progress, ) - - - - diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index a7667c31..76039313 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,5 +1,6 @@ import argparse import collections +import itertools from argparse import Action, ArgumentParser from borgmatic.config import collect @@ -75,14 +76,20 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): remaining_arguments.remove(item) try: - arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed + arguments[canonical_name] = ( + None if canonical_name in subcommand_parsers_mapping else parsed + ) except UnboundLocalError: pass - + for argument in arguments: if not arguments[argument]: - if not any(subcommand in arguments for subcommand in subcommand_parsers_mapping[argument]): - raise ValueError(f"Missing subcommand for {argument}. Expected one of {subcommand_parsers_mapping[argument]}") + if not any( + subcommand in arguments for subcommand in subcommand_parsers_mapping[argument] + ): + raise ValueError( + f'Missing subcommand for {argument}. Expected one of {subcommand_parsers_mapping[argument]}' + ) # If no actions are explicitly requested, assume defaults. if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments: @@ -93,6 +100,10 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): remaining_arguments = list(unparsed_arguments) + # Now ask each subparser, one by one, to greedily consume arguments, from last to first. This + # allows subparsers to consume arguments before their parent subparsers do. + remaining_subparser_arguments = [] + # Now ask each subparser, one by one, to greedily consume arguments, from last to first. This # allows subparsers to consume arguments before their parent subparsers do. for subparser_name, subparser in reversed(subparsers.items()): @@ -100,9 +111,23 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): continue subparser = subparsers[subparser_name] - unused_parsed, remaining_arguments = subparser.parse_known_args( - [argument for argument in remaining_arguments if argument != subparser_name] + unused_parsed, remaining = subparser.parse_known_args( + [argument for argument in unparsed_arguments if argument != subparser_name] ) + remaining_subparser_arguments.append(remaining) + + # Determine the remaining arguments that no subparsers have consumed. + if remaining_subparser_arguments: + remaining_arguments = [ + argument + for argument in dict.fromkeys( + itertools.chain.from_iterable(remaining_subparser_arguments) + ).keys() + if all( + argument in subparser_arguments + for subparser_arguments in remaining_subparser_arguments + ) + ] # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the # "borg" action. @@ -551,9 +576,7 @@ def make_parsers(): ) config_group = config_parser.add_argument_group('config arguments') - config_group.add_argument( - '-h', '--help', action='help', help='Show this help message and exit' - ) + config_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') config_subparsers = config_parser.add_subparsers( title='config subcommands', @@ -568,7 +591,9 @@ def make_parsers(): description='Extract just the config files that were used to create a borgmatic repository during the "create" operation', add_help=False, ) - config_bootstrap_group = config_bootstrap_parser.add_argument_group('config bootstrap arguments') + config_bootstrap_group = config_bootstrap_parser.add_argument_group( + 'config bootstrap arguments' + ) config_bootstrap_group.add_argument( '--repository', help='Path of repository to extract config files from', @@ -579,7 +604,9 @@ def make_parsers(): help='Path that stores the config files used to create an archive, and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic', ) config_bootstrap_group.add_argument( - '--archive', help='Name of archive to extract config files from, defaults to "latest"', default='latest' + '--archive', + help='Name of archive to extract config files from, defaults to "latest"', + default='latest', ) config_bootstrap_group.add_argument( '--destination', @@ -947,7 +974,9 @@ def make_parsers(): ) borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - merged_subparsers = argparse._SubParsersAction(None, None, metavar=None, dest='merged', parser_class=None) + merged_subparsers = argparse._SubParsersAction( + None, None, metavar=None, dest='merged', parser_class=None + ) for name, subparser in subparsers.choices.items(): merged_subparsers._name_parser_map[name] = subparser @@ -957,7 +986,7 @@ def make_parsers(): merged_subparsers._name_parser_map[name] = subparser subparser._name_parser_map = merged_subparsers._name_parser_map - return top_level_parser, merged_subparsers + return top_level_parser, merged_subparsers def parse_arguments(*unparsed_arguments): @@ -967,12 +996,15 @@ def parse_arguments(*unparsed_arguments): ''' top_level_parser, subparsers = make_parsers() - arguments, remaining_arguments = parse_subparser_arguments( unparsed_arguments, subparsers.choices ) - if 'bootstrap' in arguments.keys() and 'config' in arguments.keys() and len(arguments.keys()) > 2: + if ( + 'bootstrap' in arguments.keys() + and 'config' in arguments.keys() + and len(arguments.keys()) > 2 + ): raise ValueError( 'The bootstrap action cannot be combined with other actions. Please run it separately.' ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index ddfb9898..b3a86d98 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -18,10 +18,10 @@ import borgmatic.actions.borg import borgmatic.actions.break_lock import borgmatic.actions.check import borgmatic.actions.compact +import borgmatic.actions.config.bootstrap import borgmatic.actions.create import borgmatic.actions.export_tar import borgmatic.actions.extract -import borgmatic.actions.config.bootstrap import borgmatic.actions.info import borgmatic.actions.list import borgmatic.actions.mount @@ -622,12 +622,14 @@ def collect_configuration_run_summary_logs(configs, arguments): except ValueError as error: yield from log_error_records(str(error)) return - + if 'bootstrap' in arguments: # no configuration file is needed for bootstrap local_borg_version = borg_version.local_borg_version({}, 'borg') try: - borgmatic.actions.config.bootstrap.run_bootstrap(arguments['bootstrap'], arguments['global'], local_borg_version) + borgmatic.actions.config.bootstrap.run_bootstrap( + arguments['bootstrap'], arguments['global'], local_borg_version + ) yield logging.makeLogRecord( dict( levelno=logging.INFO, From f82631e3bbb725078e416a1e910dcf63535597b8 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 7 Jun 2023 00:56:19 +0530 Subject: [PATCH 241/344] tests for arguments.py --- tests/integration/commands/test_arguments.py | 6 ++++++ tests/unit/commands/test_arguments.py | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 4990fc4f..01e10fc6 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -297,6 +297,12 @@ def test_parse_arguments_disallows_paths_unless_action_consumes_it(): with pytest.raises(SystemExit): module.parse_arguments('--config', 'myconfig', '--path', 'test') +def test_parse_arguments_disallows_other_actions_with_config_bootstrap(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('config', 'bootstrap', '--repository', 'test.borg', 'list') + def test_parse_arguments_allows_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 9354cf5e..fafec390 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -1,5 +1,6 @@ import collections +import pytest from flexmock import flexmock from borgmatic.commands import arguments as module @@ -164,3 +165,13 @@ def test_parse_subparser_arguments_parses_borg_options_and_skips_other_subparser assert arguments == {'borg': action_namespace} assert arguments['borg'].options == ['list'] assert remaining_arguments == [] + + +def test_parse_subparser_arguments_raises_error_when_no_subparser_is_specified(): + action_namespace = flexmock(options=[]) + subparsers = { + 'config': flexmock(parse_known_args=lambda arguments: (action_namespace, ['config'])), + } + + with pytest.raises(ValueError): + module.parse_subparser_arguments(('config',), subparsers) From 2d761dd86b55f5ad9994af0e707a8c3984031236 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 7 Jun 2023 01:43:01 +0530 Subject: [PATCH 242/344] coverage at 100 --- tests/integration/commands/test_arguments.py | 1 + tests/unit/actions/config/test_bootstrap.py | 33 ++++++++++++++++++++ tests/unit/actions/test_create.py | 17 ++++++++++ tests/unit/commands/test_borgmatic.py | 16 ++++++++++ 4 files changed, 67 insertions(+) create mode 100644 tests/unit/actions/config/test_bootstrap.py diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 01e10fc6..26b94c66 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -297,6 +297,7 @@ def test_parse_arguments_disallows_paths_unless_action_consumes_it(): with pytest.raises(SystemExit): module.parse_arguments('--config', 'myconfig', '--path', 'test') + def test_parse_arguments_disallows_other_actions_with_config_bootstrap(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py new file mode 100644 index 00000000..d0423dfd --- /dev/null +++ b/tests/unit/actions/config/test_bootstrap.py @@ -0,0 +1,33 @@ +from flexmock import flexmock + +from borgmatic.actions.config import bootstrap as module + + +def test_run_bootstrap(): + bootstrap_arguments = flexmock( + repository='repo', + archive='archive', + destination='dest', + strip_components=1, + progress=False, + borgmatic_source_directory='/borgmatic', + ) + global_arguments = flexmock( + dry_run=False, + ) + local_borg_version = flexmock() + extract_process = flexmock( + stdout=flexmock( + read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}', + ), + ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + extract_process + ) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + 'archive' + ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + extract_process + ) + module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index f6d9acba..39b4c588 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -103,3 +103,20 @@ def test_run_create_bails_if_repository_does_not_match(): remote_path=None, ) ) + + +def test_create_borgmatic_manifest_creates_manifest_file(): + flexmock(module.os.path).should_receive('expanduser').and_return('/home/user') + flexmock(module.os.path).should_receive('join').and_return('/home/user/bootstrap/manifest.json') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.os).should_receive('makedirs').and_return(True) + + flexmock(module.json).should_receive('dump').and_return(True) + + module.create_borgmatic_manifest({}, 'test.yaml', False) + + +def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run(): + flexmock(module.os.path).should_receive('expanduser').never() + + module.create_borgmatic_manifest({}, 'test.yaml', True) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index bd98c01f..0df79a69 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -987,6 +987,22 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): assert {log.levelno for log in logs} == {logging.INFO} +def test_collect_configuration_run_summary_logs_info_for_success_with_bootstrap(): + flexmock(module.validate).should_receive('guard_single_repository_selected').never() + flexmock(module.validate).should_receive('guard_configuration_contains_repository').never() + flexmock(module).should_receive('run_configuration').never() + flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap') + arguments = { + 'bootstrap': flexmock(repository='repo'), + 'global': flexmock(monitoring_verbosity=1, dry_run=False), + } + + logs = tuple( + module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + ) + assert {log.levelno for log in logs} == {logging.INFO} + + def test_collect_configuration_run_summary_logs_extract_with_repository_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise( ValueError From dc56fd33a01e5f437f1bab9663bca2f0266e4b35 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 7 Jun 2023 01:47:16 +0530 Subject: [PATCH 243/344] formatting --- borgmatic/commands/arguments.py | 9 ++------- borgmatic/commands/borgmatic.py | 8 +++++++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 76039313..5e5cba6d 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -75,12 +75,7 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): if item in subparsers: remaining_arguments.remove(item) - try: - arguments[canonical_name] = ( - None if canonical_name in subcommand_parsers_mapping else parsed - ) - except UnboundLocalError: - pass + arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed for argument in arguments: if not arguments[argument]: @@ -153,7 +148,7 @@ class Extend_action(Action): items = getattr(namespace, self.dest, None) if items: - items.extend(values) + items.extend(values) # pragma: no cover else: setattr(namespace, self.dest, list(values)) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b3a86d98..3fd73c46 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -637,7 +637,13 @@ def collect_configuration_run_summary_logs(configs, arguments): msg='Bootstrap successful', ) ) - except (CalledProcessError, ValueError, OSError, json.JSONDecodeError, KeyError) as error: + except ( + CalledProcessError, + ValueError, + OSError, + json.JSONDecodeError, + KeyError, + ) as error: # pragma: no cover yield from log_error_records('Error running bootstrap', error) return From dcb90bba50561a9ad204adef15fb0fcaaecc6aaf Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 7 Jun 2023 23:56:02 +0530 Subject: [PATCH 244/344] some tests remaining --- tests/unit/actions/config/test_bootstrap.py | 33 +++++++++++++++++---- tests/unit/actions/test_create.py | 32 ++++++++++++++++++-- tests/unit/commands/test_borgmatic.py | 18 +++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index d0423dfd..a1e255b2 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -3,7 +3,33 @@ from flexmock import flexmock from borgmatic.actions.config import bootstrap as module -def test_run_bootstrap(): +def test_get_config_paths_returns_list_of_config_paths(): + bootstrap_arguments = flexmock( + borgmatic_source_directory=None, + repository='repo', + archive='archive', + ) + global_arguments = flexmock( + dry_run=False, + ) + local_borg_version = flexmock() + extract_process = flexmock( + stdout=flexmock( + read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}', + ), + ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + extract_process + ) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + 'archive' + ) + assert module.get_config_paths( + bootstrap_arguments, global_arguments, local_borg_version + ) == ['/borgmatic/config.yaml'] + + +def test_run_bootstrap_does_not_raise(): bootstrap_arguments = flexmock( repository='repo', archive='archive', @@ -23,11 +49,8 @@ def test_run_bootstrap(): ) flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( extract_process - ) + ).twice() flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( 'archive' ) - flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( - extract_process - ) module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 39b4c588..bfb1c09d 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -1,3 +1,4 @@ +import sys from flexmock import flexmock from borgmatic.actions import create as module @@ -7,6 +8,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() + flexmock(module).should_receive('create_borgmatic_manifest').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( @@ -45,6 +47,7 @@ def test_run_create_runs_with_selected_repository(): 'repositories_match' ).once().and_return(True) flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() + flexmock(module).should_receive('create_borgmatic_manifest').once() create_arguments = flexmock( repository=flexmock(), progress=flexmock(), @@ -78,6 +81,7 @@ def test_run_create_bails_if_repository_does_not_match(): 'repositories_match' ).once().and_return(False) flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + flexmock(module).should_receive('create_borgmatic_manifest').never() create_arguments = flexmock( repository=flexmock(), progress=flexmock(), @@ -106,16 +110,40 @@ def test_run_create_bails_if_repository_does_not_match(): def test_create_borgmatic_manifest_creates_manifest_file(): - flexmock(module.os.path).should_receive('expanduser').and_return('/home/user') - flexmock(module.os.path).should_receive('join').and_return('/home/user/bootstrap/manifest.json') flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.os).should_receive('makedirs').and_return(True) + flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0') flexmock(module.json).should_receive('dump').and_return(True) module.create_borgmatic_manifest({}, 'test.yaml', False) +def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_source_directory(): + flexmock(module.os.path).should_receive('join').with_args( + '/borgmatic', 'bootstrap', 'manifest.json' + ).and_return('/borgmatic/bootstrap/manifest.json') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.os).should_receive('makedirs').and_return(True) + + flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0') + flexmock(sys.modules['builtins']).should_receive('open').with_args( + '/borgmatic/bootstrap/manifest.json', 'w' + ).and_return( + flexmock( + __enter__=lambda *args: flexmock( + write=lambda *args: None, close=lambda *args: None + ), + __exit__=lambda *args: None, + ) + ) + flexmock(module.json).should_receive('dump').and_return(True) + + module.create_borgmatic_manifest( + {'borgmatic_source_directory': '/borgmatic'}, 'test.yaml', False + ) + + def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run(): flexmock(module.os.path).should_receive('expanduser').never() diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 0df79a69..68884925 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1002,6 +1002,24 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_bootstrap( ) assert {log.levelno for log in logs} == {logging.INFO} +def test_collect_configuration_run_summary_logs_error_on_bootstrap_failure(): + flexmock(module.validate).should_receive('guard_single_repository_selected').never() + flexmock(module.validate).should_receive('guard_configuration_contains_repository').never() + flexmock(module).should_receive('run_configuration').never() + flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').and_raise( + ValueError + ) + arguments = { + 'bootstrap': flexmock(repository='repo'), + 'global': flexmock(monitoring_verbosity=1, dry_run=False), + } + + logs = tuple( + module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + def test_collect_configuration_run_summary_logs_extract_with_repository_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise( From 8384eaefb1c42068f9dffe99a235f242c0f9cf5b Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 8 Jun 2023 00:07:36 +0530 Subject: [PATCH 245/344] reformat --- tests/unit/actions/config/test_bootstrap.py | 6 +-- tests/unit/actions/test_create.py | 5 +-- tests/unit/borg/test_create.py | 45 +++++++++++++++++++++ tests/unit/commands/test_borgmatic.py | 1 + 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index a1e255b2..d1b21511 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -24,9 +24,9 @@ def test_get_config_paths_returns_list_of_config_paths(): flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( 'archive' ) - assert module.get_config_paths( - bootstrap_arguments, global_arguments, local_borg_version - ) == ['/borgmatic/config.yaml'] + assert module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) == [ + '/borgmatic/config.yaml' + ] def test_run_bootstrap_does_not_raise(): diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index bfb1c09d..5846b8ad 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -1,4 +1,5 @@ import sys + from flexmock import flexmock from borgmatic.actions import create as module @@ -131,9 +132,7 @@ def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_s '/borgmatic/bootstrap/manifest.json', 'w' ).and_return( flexmock( - __enter__=lambda *args: flexmock( - write=lambda *args: None, close=lambda *args: None - ), + __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None), __exit__=lambda *args: None, ) ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index efabd0ff..157ae9de 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -586,6 +586,51 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert ) +def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sources_and_config_paths(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('deduplicate_directories').and_return( + ('foo', 'bar', '/etc/borgmatic/config.yaml') + ) + flexmock(module).should_receive('map_directories_to_devices').and_return({}) + flexmock(module).should_receive('expand_directories').and_return(()) + flexmock(module).should_receive('pattern_root_directories').and_return([]) + flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) + flexmock(module).should_receive('expand_home_directories').and_return(()) + flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module).should_receive('ensure_files_readable') + flexmock(module).should_receive('make_pattern_flags').and_return(()) + flexmock(module).should_receive('make_exclude_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',) + ) + environment = {'BORG_THINGY': 'YUP'} + flexmock(module.environment).should_receive('make_environment').and_return(environment) + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('/etc/borgmatic/config.yaml',), + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=environment, + ) + + module.create_archive( + dry_run=False, + repository_path='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + }, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']), + ) + + def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 68884925..1a20683f 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1002,6 +1002,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_bootstrap( ) assert {log.levelno for log in logs} == {logging.INFO} + def test_collect_configuration_run_summary_logs_error_on_bootstrap_failure(): flexmock(module.validate).should_receive('guard_single_repository_selected').never() flexmock(module.validate).should_receive('guard_configuration_contains_repository').never() From f90d30e0e19a4e56ac58d51583be5b48f180c677 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 8 Jun 2023 00:08:39 +0530 Subject: [PATCH 246/344] remove duplicate comments --- borgmatic/commands/arguments.py | 2 -- borgmatic/commands/borgmatic.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 5e5cba6d..3a527799 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -99,8 +99,6 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): # allows subparsers to consume arguments before their parent subparsers do. remaining_subparser_arguments = [] - # Now ask each subparser, one by one, to greedily consume arguments, from last to first. This - # allows subparsers to consume arguments before their parent subparsers do. for subparser_name, subparser in reversed(subparsers.items()): if subparser_name not in arguments.keys(): continue diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 3fd73c46..766073cf 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -643,7 +643,7 @@ def collect_configuration_run_summary_logs(configs, arguments): OSError, json.JSONDecodeError, KeyError, - ) as error: # pragma: no cover + ) as error: yield from log_error_records('Error running bootstrap', error) return From 6475345a8fbaa9b3d9aaeb7444075829c49343ab Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 8 Jun 2023 01:02:43 +0530 Subject: [PATCH 247/344] attempt to test parse_subparser_arguments --- borgmatic/commands/arguments.py | 58 +++++++++++++++------------ tests/unit/commands/test_arguments.py | 19 +++++++++ 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 3a527799..b68590b8 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -28,6 +28,37 @@ SUBPARSER_ALIASES = { } +def parse_subparser_arguments(arguments, unparsed_arguments, subparsers): + remaining_subparser_arguments = [] + + for subparser_name, subparser in reversed(subparsers.items()): + if subparser_name not in arguments.keys(): + continue + + subparser = subparsers[subparser_name] + unused_parsed, remaining = subparser.parse_known_args( + [argument for argument in unparsed_arguments if argument != subparser_name] + ) + remaining_subparser_arguments.append(remaining) + + # Determine the remaining arguments that no subparsers have consumed. + if remaining_subparser_arguments: + remaining_arguments = [ + argument + for argument in dict.fromkeys( + itertools.chain.from_iterable(remaining_subparser_arguments) + ).keys() + if all( + argument in subparser_arguments + for subparser_arguments in remaining_subparser_arguments + ) + ] + else: + remaining_arguments = [] + + return remaining_arguments + + def parse_subparser_arguments(unparsed_arguments, subparsers): ''' Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser @@ -97,30 +128,7 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): # Now ask each subparser, one by one, to greedily consume arguments, from last to first. This # allows subparsers to consume arguments before their parent subparsers do. - remaining_subparser_arguments = [] - - for subparser_name, subparser in reversed(subparsers.items()): - if subparser_name not in arguments.keys(): - continue - - subparser = subparsers[subparser_name] - unused_parsed, remaining = subparser.parse_known_args( - [argument for argument in unparsed_arguments if argument != subparser_name] - ) - remaining_subparser_arguments.append(remaining) - - # Determine the remaining arguments that no subparsers have consumed. - if remaining_subparser_arguments: - remaining_arguments = [ - argument - for argument in dict.fromkeys( - itertools.chain.from_iterable(remaining_subparser_arguments) - ).keys() - if all( - argument in subparser_arguments - for subparser_arguments in remaining_subparser_arguments - ) - ] + remaining_arguments = parse_subparser_arguments(arguments, unparsed_arguments, subparsers) # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the # "borg" action. @@ -973,11 +981,9 @@ def make_parsers(): for name, subparser in subparsers.choices.items(): merged_subparsers._name_parser_map[name] = subparser - subparser._name_parser_map = merged_subparsers._name_parser_map for name, subparser in config_subparsers.choices.items(): merged_subparsers._name_parser_map[name] = subparser - subparser._name_parser_map = merged_subparsers._name_parser_map return top_level_parser, merged_subparsers diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index fafec390..c532022e 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -175,3 +175,22 @@ def test_parse_subparser_arguments_raises_error_when_no_subparser_is_specified() with pytest.raises(ValueError): module.parse_subparser_arguments(('config',), subparsers) + + +@pytest.mark.parametrize( + 'arguments, unparsed_arguments, subparsers, expected_remaining_arguments', + [ + ( + {'action': flexmock()}, + ['--verbosity', 'lots'], + {'action': flexmock(parse_known_args=lambda arguments: (flexmock(), ['--verbosity', 'lots']))}, + ['--verbosity', 'lots'], + ), + ], +) +def test_get_remaining_arguments_returns_expected_remaining_arguments( + arguments, unparsed_arguments, subparsers, expected_remaining_arguments +): + remaining_arguments = module.get_remaining_arguments(arguments, unparsed_arguments, subparsers) + + assert remaining_arguments == expected_remaining_arguments \ No newline at end of file From 3315555d06e4690a9905a42de77dae870db47372 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 9 Jun 2023 00:21:41 +0530 Subject: [PATCH 248/344] cleaner test --- borgmatic/commands/arguments.py | 29 +++++++++++----------- tests/unit/commands/test_arguments.py | 35 ++++++++++++++++++--------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index b68590b8..db506741 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -28,19 +28,7 @@ SUBPARSER_ALIASES = { } -def parse_subparser_arguments(arguments, unparsed_arguments, subparsers): - remaining_subparser_arguments = [] - - for subparser_name, subparser in reversed(subparsers.items()): - if subparser_name not in arguments.keys(): - continue - - subparser = subparsers[subparser_name] - unused_parsed, remaining = subparser.parse_known_args( - [argument for argument in unparsed_arguments if argument != subparser_name] - ) - remaining_subparser_arguments.append(remaining) - +def get_unparsable_arguments(remaining_subparser_arguments): # Determine the remaining arguments that no subparsers have consumed. if remaining_subparser_arguments: remaining_arguments = [ @@ -128,7 +116,20 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): # Now ask each subparser, one by one, to greedily consume arguments, from last to first. This # allows subparsers to consume arguments before their parent subparsers do. - remaining_arguments = parse_subparser_arguments(arguments, unparsed_arguments, subparsers) + remaining_subparser_arguments = [] + + for subparser_name, subparser in reversed(subparsers.items()): + if subparser_name not in arguments.keys(): + continue + + subparser = subparsers[subparser_name] + unused_parsed, remaining = subparser.parse_known_args( + [argument for argument in unparsed_arguments if argument != subparser_name] + ) + remaining_subparser_arguments.append(remaining) + + if remaining_subparser_arguments: + remaining_arguments = get_unparsable_arguments(remaining_subparser_arguments) # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the # "borg" action. diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index c532022e..ef8fd1ea 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -178,19 +178,32 @@ def test_parse_subparser_arguments_raises_error_when_no_subparser_is_specified() @pytest.mark.parametrize( - 'arguments, unparsed_arguments, subparsers, expected_remaining_arguments', - [ + 'arguments, expected', + [ ( - {'action': flexmock()}, - ['--verbosity', 'lots'], - {'action': flexmock(parse_known_args=lambda arguments: (flexmock(), ['--verbosity', 'lots']))}, - ['--verbosity', 'lots'], + ( + ('--latest', 'archive', 'prune', 'extract', 'list', '--test-flag'), + ('--latest', 'archive', 'check', 'extract', 'list', '--test-flag'), + ('prune', 'check', 'list', '--test-flag'), + ('prune', 'check', 'extract', '--test-flag'), + ), + [ + '--test-flag', + ], ), + ( + ( + ('--latest', 'archive', 'prune', 'extract', 'list'), + ('--latest', 'archive', 'check', 'extract', 'list'), + ('prune', 'check', 'list'), + ('prune', 'check', 'extract'), + ), + [], + ), + ((), []), ], ) -def test_get_remaining_arguments_returns_expected_remaining_arguments( - arguments, unparsed_arguments, subparsers, expected_remaining_arguments +def test_get_unparsable_arguments_returns_remaining_arguments_that_no_subparser_can_parse( + arguments, expected ): - remaining_arguments = module.get_remaining_arguments(arguments, unparsed_arguments, subparsers) - - assert remaining_arguments == expected_remaining_arguments \ No newline at end of file + assert module.get_unparsable_arguments(arguments) == expected From 425f260a22f0c74b214b3cfd1d59ea8da4c96f2f Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 9 Jun 2023 04:15:18 +0530 Subject: [PATCH 249/344] test parser merging --- tests/integration/commands/test_arguments.py | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 26b94c66..8c96ac97 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -1,3 +1,5 @@ +import argparse + import pytest from flexmock import flexmock @@ -530,3 +532,33 @@ def test_parse_arguments_extract_with_check_only_extract_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract') + + +def test_merging_two_subparser_collections_merges_their_choices(): + top_level_parser = argparse.ArgumentParser() + + subparsers = top_level_parser.add_subparsers() + + subparser1 = subparsers.add_parser('subparser1') + + subparser2 = subparsers.add_parser('subparser2') + + subsubparsers = subparser2.add_subparsers() + + subsubparser1 = subsubparsers.add_parser('subsubparser1') + + merged_subparsers = argparse._SubParsersAction( + None, None, metavar=None, dest='merged', parser_class=None + ) + + for name, subparser in subparsers.choices.items(): + merged_subparsers._name_parser_map[name] = subparser + + for name, subparser in subsubparsers.choices.items(): + merged_subparsers._name_parser_map[name] = subparser + + assert merged_subparsers.choices == { + 'subparser1': subparser1, + 'subparser2': subparser2, + 'subsubparser1': subsubparser1, + } From 197920d9efa9e0e823e1bdf7e87ff2b56747867e Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 9 Jun 2023 17:31:57 +0530 Subject: [PATCH 250/344] improve tests and some docstrings. --- borgmatic/actions/config/bootstrap.py | 10 +++++++ borgmatic/commands/arguments.py | 29 ++++++++++++++------ borgmatic/commands/borgmatic.py | 1 + tests/integration/commands/test_arguments.py | 9 +----- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 5b323451..9254da6d 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -12,6 +12,16 @@ logger = logging.getLogger(__name__) def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): + ''' + Given: + The bootstrap arguments, which include the repository and archive name, borgmatic source directory, + destination directory, and whether to strip components. + The global arguments, which include the dry run flag + and the local borg version, + Return: + The config paths from the manifest.json file in the borgmatic source directory after extracting it from the + repository. + ''' borgmatic_source_directory = ( bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index db506741..c39198c3 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -29,7 +29,9 @@ SUBPARSER_ALIASES = { def get_unparsable_arguments(remaining_subparser_arguments): - # Determine the remaining arguments that no subparsers have consumed. + ''' + Determine the remaining arguments that no subparsers have consumed. + ''' if remaining_subparser_arguments: remaining_arguments = [ argument @@ -590,7 +592,7 @@ def make_parsers(): 'bootstrap', aliases=SUBPARSER_ALIASES['config_bootstrap'], help='Extract the config files used to create a borgmatic repository', - description='Extract just the config files that were used to create a borgmatic repository during the "create" operation', + description='Extract config files that were used to create a borgmatic repository during the "create" operation', add_help=False, ) config_bootstrap_group = config_bootstrap_parser.add_argument_group( @@ -603,7 +605,7 @@ def make_parsers(): ) config_bootstrap_group.add_argument( '--borgmatic-source-directory', - help='Path that stores the config files used to create an archive, and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic', + help='Path that stores the config files used to create an archive and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic', ) config_bootstrap_group.add_argument( '--archive', @@ -980,15 +982,26 @@ def make_parsers(): None, None, metavar=None, dest='merged', parser_class=None ) - for name, subparser in subparsers.choices.items(): - merged_subparsers._name_parser_map[name] = subparser - - for name, subparser in config_subparsers.choices.items(): - merged_subparsers._name_parser_map[name] = subparser + merged_subparsers = merge_subparsers(subparsers, config_subparsers) return top_level_parser, merged_subparsers +def merge_subparsers(*subparsers): + ''' + Merge multiple subparsers into a single subparser. + ''' + merged_subparsers = argparse._SubParsersAction( + None, None, metavar=None, dest='merged', parser_class=None + ) + + for subparser in subparsers: + for name, subparser in subparser.choices.items(): + merged_subparsers._name_parser_map[name] = subparser + + return merged_subparsers + + def parse_arguments(*unparsed_arguments): ''' Given command-line arguments with which this script was invoked, parse the arguments and return diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 766073cf..4ad81362 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -645,6 +645,7 @@ def collect_configuration_run_summary_logs(configs, arguments): KeyError, ) as error: yield from log_error_records('Error running bootstrap', error) + return if not configs: diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 8c96ac97..60fe884c 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -538,24 +538,17 @@ def test_merging_two_subparser_collections_merges_their_choices(): top_level_parser = argparse.ArgumentParser() subparsers = top_level_parser.add_subparsers() - subparser1 = subparsers.add_parser('subparser1') subparser2 = subparsers.add_parser('subparser2') - subsubparsers = subparser2.add_subparsers() - subsubparser1 = subsubparsers.add_parser('subsubparser1') merged_subparsers = argparse._SubParsersAction( None, None, metavar=None, dest='merged', parser_class=None ) - for name, subparser in subparsers.choices.items(): - merged_subparsers._name_parser_map[name] = subparser - - for name, subparser in subsubparsers.choices.items(): - merged_subparsers._name_parser_map[name] = subparser + merged_subparsers = module.merge_subparsers(subparsers, subsubparsers) assert merged_subparsers.choices == { 'subparser1': subparser1, From d370ff958d04bc43a31f38f6234430e1d586609c Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 10 Jun 2023 01:05:34 +0530 Subject: [PATCH 251/344] mock expand directories thrice --- tests/unit/borg/test_create.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 157ae9de..c798c446 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -594,7 +594,11 @@ def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sourc ('foo', 'bar', '/etc/borgmatic/config.yaml') ) flexmock(module).should_receive('map_directories_to_devices').and_return({}) - flexmock(module).should_receive('expand_directories').and_return(()) + flexmock(module).should_receive('expand_directories').with_args([]).and_return(()) + flexmock(module).should_receive('expand_directories').with_args( + ('foo', 'bar', '/etc/borgmatic/config.yaml') + ).and_return(('foo', 'bar', '/etc/borgmatic/config.yaml')) + flexmock(module).should_receive('expand_directories').with_args([]).and_return(()) flexmock(module).should_receive('pattern_root_directories').and_return([]) flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) flexmock(module).should_receive('expand_home_directories').and_return(()) From d84f1ec616971638290fd8641b06db41befb9218 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 10 Jun 2023 14:52:00 -0700 Subject: [PATCH 252/344] Add bootstrap action to NEWS and make post-PR tweaks (#697). --- NEWS | 4 ++++ borgmatic/actions/config/__init__.py | 0 borgmatic/commands/arguments.py | 4 ---- setup.py | 2 +- tests/unit/actions/test_create.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 borgmatic/actions/config/__init__.py diff --git a/NEWS b/NEWS index 87252c7b..54d6d69e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.7.15.dev0 + * #697: Extract borgmatic configuration from backup via "borgmatic bootstrap" action—even when + borgmatic has no configuration yet! + 1.7.14 * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file, or monitoring), so not even errors are shown. diff --git a/borgmatic/actions/config/__init__.py b/borgmatic/actions/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 64858521..ca4c2dbc 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1086,10 +1086,6 @@ def make_parsers(): ) borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - merged_subparsers = argparse._SubParsersAction( - None, None, metavar=None, dest='merged', parser_class=None - ) - merged_subparsers = merge_subparsers(subparsers, config_subparsers) return top_level_parser, merged_subparsers diff --git a/setup.py b/setup.py index df0718de..8fb11b1f 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.14' +VERSION = '1.7.15.dev0' setup( diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 5846b8ad..7eeca8b2 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -115,7 +115,7 @@ def test_create_borgmatic_manifest_creates_manifest_file(): flexmock(module.os).should_receive('makedirs').and_return(True) flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0') - flexmock(module.json).should_receive('dump').and_return(True) + flexmock(module.json).should_receive('dump').and_return(True).once() module.create_borgmatic_manifest({}, 'test.yaml', False) @@ -136,7 +136,7 @@ def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_s __exit__=lambda *args: None, ) ) - flexmock(module.json).should_receive('dump').and_return(True) + flexmock(module.json).should_receive('dump').and_return(True).once() module.create_borgmatic_manifest( {'borgmatic_source_directory': '/borgmatic'}, 'test.yaml', False From 0f9756e7395d49c8d75ac8796feeac846814fb22 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 10 Jun 2023 15:17:18 -0700 Subject: [PATCH 253/344] Fix failing test and add "bootstrap" action to CLI reference docs (#697). --- borgmatic/actions/create.py | 4 ++-- borgmatic/commands/arguments.py | 2 +- docs/Dockerfile | 4 ++-- tests/unit/actions/test_create.py | 11 +++++++++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 4d83634b..1bacf73b 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -8,11 +8,11 @@ except ModuleNotFoundError: # pragma: nocover import importlib.metadata as importlib_metadata import borgmatic.borg.create +import borgmatic.borg.state import borgmatic.config.validate import borgmatic.hooks.command import borgmatic.hooks.dispatch import borgmatic.hooks.dump -from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def create_borgmatic_manifest(location, config_paths, dry_run): return borgmatic_source_directory = location.get( - 'borgmatic_source_directory', DEFAULT_BORGMATIC_SOURCE_DIRECTORY + 'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY ) borgmatic_manifest_path = os.path.expanduser( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index ca4c2dbc..30ac3f9c 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -632,7 +632,7 @@ def make_parsers(): 'bootstrap', aliases=SUBPARSER_ALIASES['config_bootstrap'], help='Extract the config files used to create a borgmatic repository', - description='Extract config files that were used to create a borgmatic repository during the "create" operation', + description='Extract config files that were used to create a borgmatic repository during the "create" action', add_help=False, ) config_bootstrap_group = config_bootstrap_parser.add_argument_group( diff --git a/docs/Dockerfile b/docs/Dockerfile index b612596e..750d4e53 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -4,9 +4,9 @@ COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ - && for action in rcreate transfer create prune compact check extract export-tar mount umount restore rlist list rinfo info break-lock borg; do \ + && for action in rcreate transfer create prune compact check extract config "config bootstrap" export-tar mount umount restore rlist list rinfo info break-lock borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ - && borgmatic "$action" --help >> /command-line.txt; done + && borgmatic $action --help >> /command-line.txt; done FROM docker.io/node:19.5.0-alpine as html diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 7eeca8b2..de94fd7e 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -111,10 +111,21 @@ def test_run_create_bails_if_repository_does_not_match(): def test_create_borgmatic_manifest_creates_manifest_file(): + flexmock(module.os.path).should_receive('join').with_args( + module.borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY, 'bootstrap', 'manifest.json' + ).and_return('/home/user/.borgmatic/bootstrap/manifest.json') flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.os).should_receive('makedirs').and_return(True) flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0') + flexmock(sys.modules['builtins']).should_receive('open').with_args( + '/home/user/.borgmatic/bootstrap/manifest.json', 'w' + ).and_return( + flexmock( + __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None), + __exit__=lambda *args: None, + ) + ) flexmock(module.json).should_receive('dump').and_return(True).once() module.create_borgmatic_manifest({}, 'test.yaml', False) From beb899d6fbf0caa64f8548075351e544c8697217 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 10 Jun 2023 15:50:11 -0700 Subject: [PATCH 254/344] Make user-facing manifest loading error messages a little friendlier (#697). --- borgmatic/actions/config/bootstrap.py | 24 ++++++- borgmatic/commands/borgmatic.py | 4 +- tests/unit/actions/config/test_bootstrap.py | 71 +++++++++++++++++++++ 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 9254da6d..3e4eb150 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -21,6 +21,9 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): Return: The config paths from the manifest.json file in the borgmatic source directory after extracting it from the repository. + + Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the + expected configuration path data. ''' borgmatic_source_directory = ( bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY @@ -46,14 +49,31 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): extract_to_stdout=True, ) - manifest_data = json.loads(extract_process.stdout.read()) + manifest_json = extract_process.stdout.read() + if not manifest_json: + raise ValueError( + 'Cannot read configuration paths from archive due to missing bootstrap manifest' + ) - return manifest_data['config_paths'] + try: + manifest_data = json.loads(manifest_json) + except json.JSONDecodeError as error: + raise ValueError( + f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}' + ) + + try: + return manifest_data['config_paths'] + except KeyError: + raise ValueError('Cannot read configuration paths from archive due to invalid bootstrap manifest') def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): ''' Run the "bootstrap" action for the given repository. + + Raise ValueError if the bootstrap configuration could not be loaded. + Raise CalledProcessError or OSError if Borg could not be run. ''' manifest_config_paths = get_config_paths( bootstrap_arguments, global_arguments, local_borg_version diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 69b7e8dd..6523ebbc 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -647,10 +647,8 @@ def collect_configuration_run_summary_logs(configs, arguments): CalledProcessError, ValueError, OSError, - json.JSONDecodeError, - KeyError, ) as error: - yield from log_error_records('Error running bootstrap', error) + yield from log_error_records(error) return diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index d1b21511..8c2063a5 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -1,3 +1,4 @@ +import pytest from flexmock import flexmock from borgmatic.actions.config import bootstrap as module @@ -29,6 +30,76 @@ def test_get_config_paths_returns_list_of_config_paths(): ] +def test_get_config_paths_with_missing_manifest_raises_value_error(): + bootstrap_arguments = flexmock( + borgmatic_source_directory=None, + repository='repo', + archive='archive', + ) + global_arguments = flexmock( + dry_run=False, + ) + local_borg_version = flexmock() + extract_process = flexmock(stdout=flexmock(read=lambda: '')) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + extract_process + ) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + 'archive' + ) + + with pytest.raises(ValueError): + module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) + + +def test_get_config_paths_with_broken_json_raises_value_error(): + bootstrap_arguments = flexmock( + borgmatic_source_directory=None, + repository='repo', + archive='archive', + ) + global_arguments = flexmock( + dry_run=False, + ) + local_borg_version = flexmock() + extract_process = flexmock( + stdout=flexmock(read=lambda: '{"config_paths": ["/oops'), + ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + extract_process + ) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + 'archive' + ) + + with pytest.raises(ValueError): + module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) + + +def test_get_config_paths_with_json_missing_key_raises_value_error(): + bootstrap_arguments = flexmock( + borgmatic_source_directory=None, + repository='repo', + archive='archive', + ) + global_arguments = flexmock( + dry_run=False, + ) + local_borg_version = flexmock() + extract_process = flexmock( + stdout=flexmock(read=lambda: '{}'), + ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + extract_process + ) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + 'archive' + ) + + with pytest.raises(ValueError): + module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) + + def test_run_bootstrap_does_not_raise(): bootstrap_arguments = flexmock( repository='repo', From 691d4f887af6d7b3b946669603bc91f279f8917d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 10 Jun 2023 16:02:03 -0700 Subject: [PATCH 255/344] Fix incorrect log message (#697). --- borgmatic/actions/config/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 3e4eb150..9770b0b8 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -80,7 +80,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): ) for config_path in manifest_config_paths: - logger.info('Bootstrapping config path %s', config_path) + logger.info(f'Bootstrapping config path {config_path}') borgmatic.borg.extract.extract_archive( global_arguments.dry_run, From 670bdffb3c0e9c7297d208c82ce6a0d6c12bddc6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 10 Jun 2023 19:25:49 -0700 Subject: [PATCH 256/344] Code formatting. --- borgmatic/actions/config/bootstrap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 9770b0b8..7ece946f 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -65,7 +65,9 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): try: return manifest_data['config_paths'] except KeyError: - raise ValueError('Cannot read configuration paths from archive due to invalid bootstrap manifest') + raise ValueError( + 'Cannot read configuration paths from archive due to invalid bootstrap manifest' + ) def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): From 41924f240011015f3e116c282e198ae6d08411d8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 11 Jun 2023 09:50:57 -0700 Subject: [PATCH 257/344] A little activism. --- NEWS | 4 ++-- README.md | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 54d6d69e..15344100 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ 1.7.15.dev0 - * #697: Extract borgmatic configuration from backup via "borgmatic bootstrap" action—even when - borgmatic has no configuration yet! + * #697: Extract borgmatic configuration from backup via "bootstrap" action—even when borgmatic + has no configuration yet! 1.7.14 * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file, diff --git a/README.md b/README.md index 079ec8ae..92b59b2e 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,10 @@ issues. ### Social -Check out the [Borg subreddit](https://www.reddit.com/r/BorgBackup/) for -general Borg and borgmatic discussion and support. +~~Check out the [Borg subreddit](https://www.reddit.com/r/BorgBackup/) for +general Borg and borgmatic discussion and support.~~ borgmatic supports the +ongoing [Reddit user +protests](https://www.theverge.com/2023/6/10/23756476/reddit-protest-api-changes-apollo-third-party-apps). Also follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic). From f558cb31563dcf303ca5eebc469a8e462f2ccbe6 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 12 Jun 2023 21:54:39 +0530 Subject: [PATCH 258/344] feat: allow restoring to different port/host/username --- borgmatic/config/schema.yaml | 24 ++++++++++++++++++++++++ borgmatic/hooks/postgresql.py | 11 +++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 903c0432..50abc3fa 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -763,10 +763,21 @@ properties: Database hostname to connect to. Defaults to connecting via local Unix socket. example: database.example.org + restore_hostname: + type: string + description: | + Database hostname to restore to. Defaults to + the "hostname" option. + example: database.example.org port: type: integer description: Port to connect to. Defaults to 5432. example: 5433 + restore_port: + type: integer + description: Port to restore to. Defaults to the + "port" option. + example: 5433 username: type: string description: | @@ -775,6 +786,12 @@ properties: You probably want to specify the "postgres" superuser here when the database name is "all". example: dbuser + restore_username: + type: string + description: | + Username with which to restore to the database. + Defaults to the "username" option. + example: dbuser password: type: string description: | @@ -784,6 +801,13 @@ properties: without a password or you create a ~/.pgpass file. example: trustsome1 + restore_password: + type: string + description: | + Password with which to connect to the database that + is being restored to. Defaults to the "password" + option. + example: trustsome1 format: type: string enum: ['plain', 'custom', 'directory', 'tar'] diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 3325391f..dab1e83a 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -217,10 +217,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, analyze_command = ( tuple(psql_command) + ('--no-password', '--no-psqlrc', '--quiet') - + (('--host', database['hostname']) if 'hostname' in database else ()) - + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--username', database['username']) if 'username' in database else ()) - + (('--dbname', database['name']) if not all_databases else ()) + + (('--host', database.get('restore_hostname', database.get('hostname', ())))) + + (('--port', str(database.get('restore_port', database.get('port', ())))) + + (('--username', database.get('restore_username', database.get('username', ())))) + + (('--dbname', database['name']) if not all_databases else ())) + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + ('--command', 'ANALYZE') ) @@ -245,6 +245,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, extra_environment = make_extra_environment(database) + if 'restore_password' in database: + extra_environment['PGPASSWORD'] = database['restore_password'] + logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") if dry_run: return From 8e8e64d920a8d4ad7c31dd12b7da361b616987f4 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 13 Jun 2023 23:42:50 +0530 Subject: [PATCH 259/344] add no-owner and refactor --- borgmatic/config/schema.yaml | 21 +++++++++++++++++---- borgmatic/hooks/postgresql.py | 25 +++++++++++++------------ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 50abc3fa..20fe3f97 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -789,7 +789,7 @@ properties: restore_username: type: string description: | - Username with which to restore to the database. + Username with which to restore the database. Defaults to the "username" option. example: dbuser password: @@ -804,10 +804,23 @@ properties: restore_password: type: string description: | - Password with which to connect to the database that - is being restored to. Defaults to the "password" - option. + Password with which to connect to the restore + database. Defaults to the "password" option. example: trustsome1 + no_owner: + type: boolean + description: | + Do not output commands to set ownership of + objects to match the original database. By + default, pg_dump and pg_restore issue ALTER + OWNER or SET SESSION AUTHORIZATION statements + to set ownership of created schema elements. + These statements will fail unless the initial + connection to the database is made by a superuser + (in which case they will execute as though wrapped + in SECURITY DEFINER functions). When --no-owner + is used, neither the ALTER OWNER nor SET SESSION + AUTHORIZATION statements will be emitted. format: type: string enum: ['plain', 'custom', 'directory', 'tar'] diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index dab1e83a..95ec4ace 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -23,13 +23,15 @@ def make_dump_path(location_config): # pragma: no cover ) -def make_extra_environment(database): +def make_extra_environment(database, restore=False): ''' Make the extra_environment dict from the given database configuration. ''' extra = dict() if 'password' in database: extra['PGPASSWORD'] = database['password'] + if restore and 'restore_password' in database: + extra['PGPASSWORD'] = database['restore_password'] extra['PGSSLMODE'] = database.get('ssl_mode', 'disable') if 'ssl_cert' in database: extra['PGSSLCERT'] = database['ssl_cert'] @@ -135,6 +137,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) + + (('--no-owner',) if database['no_owner'] else ()) + (('--format', dump_format) if dump_format else ()) + (('--file', dump_filename) if dump_format == 'directory' else ()) + (tuple(database['options'].split(' ')) if 'options' in database else ()) @@ -217,10 +220,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, analyze_command = ( tuple(psql_command) + ('--no-password', '--no-psqlrc', '--quiet') - + (('--host', database.get('restore_hostname', database.get('hostname', ())))) - + (('--port', str(database.get('restore_port', database.get('port', ())))) - + (('--username', database.get('restore_username', database.get('username', ())))) - + (('--dbname', database['name']) if not all_databases else ())) + + (('--host', database.get('restore_hostname', database.get('hostname'))) if 'hostname' in database else ()) + + (('--port', str(database.get('restore_port', database.get('port')))) if 'port' in database else ()) + + (('--username', database.get('restore_username', database.get('username'))) if 'username' in database else ()) + + (('--dbname', database['name']) if not all_databases else ()) + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + ('--command', 'ANALYZE') ) @@ -231,9 +234,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ('--no-password',) + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean')) + (('--dbname', database['name']) if not all_databases else ()) - + (('--host', database['hostname']) if 'hostname' in database else ()) - + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--username', database['username']) if 'username' in database else ()) + + (('--host', database.get('restore_hostname', database.get('hostname'))) if 'hostname' in database or 'restore_hostname' in database else ()) + + (('--port', str(database.get('restore_port', database.get('port')))) if 'port' in database or 'restore_port' in database else ()) + + (('--username', database.get('restore_username', database.get('username'))) if 'username' in database or 'restore_username' in database else ()) + + (('--no-owner',) if database['no_owner'] else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) + tuple( @@ -243,10 +247,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) ) - extra_environment = make_extra_environment(database) - - if 'restore_password' in database: - extra_environment['PGPASSWORD'] = database['restore_password'] + extra_environment = make_extra_environment(database, restore=True) logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") if dry_run: From 230cf6adc45a3f15cae54da11d934fe2560f8c45 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 14 Jun 2023 00:11:19 +0530 Subject: [PATCH 260/344] support command line args for hostname port username password --- borgmatic/actions/restore.py | 17 ++++++++++++++++- borgmatic/commands/arguments.py | 15 +++++++++++++++ borgmatic/hooks/postgresql.py | 30 ++++++++++++++++++++---------- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index ded83f4f..a701aa0d 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -68,15 +68,21 @@ def restore_single_database( archive_name, hook_name, database, + connection_params, ): # pragma: no cover ''' - Given (among other things) an archive name, a database hook name, and a configured database + Given (among other things) an archive name, a database hook name, the hostname, + port, username and password as connection params, and a configured database configuration dict, restore that database from the archive. ''' logger.info( f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}' ) + logger.info( + f'hostname port username password for database {database["name"]}' + ) + dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', hooks, @@ -113,6 +119,7 @@ def restore_single_database( location, global_arguments.dry_run, extract_process, + connection_params, ) @@ -308,6 +315,13 @@ def run_restore( hooks, archive_database_names, hook_name, database_name ) + connection_params = { + 'hostname': restore_arguments.hostname, + 'port': restore_arguments.port, + 'username': restore_arguments.username, + 'password': restore_arguments.password, + } + if not found_database: remaining_restore_names.setdefault(found_hook_name or hook_name, []).append( database_name @@ -327,6 +341,7 @@ def run_restore( archive_name, found_hook_name or hook_name, dict(found_database, **{'schemas': restore_arguments.schemas}), + connection_params, ) # For any database that weren't found via exact matches in the hooks configuration, try to diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6039e4fa..02f2fc3e 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -720,6 +720,21 @@ def make_parsers(): dest='schemas', help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases', ) + restore_group.add_argument( + '--hostname', + help='Database hostname to restore to. Defaults to the "restore_hostname" option in borgmatic\'s configuration', + ) + restore_group.add_argument( + '--port', help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration' + ) + restore_group.add_argument( + '--username', + help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration', + ) + restore_group.add_argument( + '--password', + help='Password with which to connect to the restore database. Defaults to the "restore_password" option in borgmatic\'s configuration', + ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' ) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 95ec4ace..89ade66f 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -22,8 +22,7 @@ def make_dump_path(location_config): # pragma: no cover location_config.get('borgmatic_source_directory'), 'postgresql_databases' ) - -def make_extra_environment(database, restore=False): +def make_extra_environment(database, restore=False, connection_params=None): ''' Make the extra_environment dict from the given database configuration. ''' @@ -32,6 +31,8 @@ def make_extra_environment(database, restore=False): extra['PGPASSWORD'] = database['password'] if restore and 'restore_password' in database: extra['PGPASSWORD'] = database['restore_password'] + if connection_params is not None and connection_params.get('password'): + extra['PGPASSWORD'] = connection_params['password'] extra['PGSSLMODE'] = database.get('ssl_mode', 'disable') if 'ssl_cert' in database: extra['PGSSLCERT'] = database['ssl_cert'] @@ -195,7 +196,7 @@ def make_database_dump_pattern( return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') -def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process): +def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params): ''' Restore the given PostgreSQL database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. @@ -205,6 +206,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, If the extract process is None, then restore the dump from the filesystem rather than from an extract stream. + + Use the given connection parameters to connect to the database. The connection parameters are + hostname, port, username, and password. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' @@ -212,6 +216,12 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, raise ValueError('The database configuration value is invalid') database = database_config[0] + + hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) + port = str(connection_params['port'] or database.get('restore_port', database.get('port'))) + username = connection_params['username'] or database.get('restore_username', database.get('username')) + password = connection_params['password'] or database.get('restore_password', database.get('password')) + all_databases = bool(database['name'] == 'all') dump_filename = dump.make_database_dump_filename( make_dump_path(location_config), database['name'], database.get('hostname') @@ -220,9 +230,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, analyze_command = ( tuple(psql_command) + ('--no-password', '--no-psqlrc', '--quiet') - + (('--host', database.get('restore_hostname', database.get('hostname'))) if 'hostname' in database else ()) - + (('--port', str(database.get('restore_port', database.get('port')))) if 'port' in database else ()) - + (('--username', database.get('restore_username', database.get('username'))) if 'username' in database else ()) + + (('--host', hostname) if 'hostname' in database or 'restore_hostname' in database or 'hostname' in connection_params else ()) + + (('--port', port) if 'port' in database or 'restore_port' in database or 'port' in connection_params else ()) + + (('--username', username) if 'username' in database or 'restore_username' in database or 'username' in connection_params else ()) + (('--dbname', database['name']) if not all_databases else ()) + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + ('--command', 'ANALYZE') @@ -234,9 +244,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ('--no-password',) + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean')) + (('--dbname', database['name']) if not all_databases else ()) - + (('--host', database.get('restore_hostname', database.get('hostname'))) if 'hostname' in database or 'restore_hostname' in database else ()) - + (('--port', str(database.get('restore_port', database.get('port')))) if 'port' in database or 'restore_port' in database else ()) - + (('--username', database.get('restore_username', database.get('username'))) if 'username' in database or 'restore_username' in database else ()) + + (('--host', hostname) if 'hostname' in database or 'restore_hostname' in database or 'hostname' in connection_params else ()) + + (('--port', port) if 'port' in database or 'restore_port' in database or 'port' in connection_params else ()) + + (('--username', username) if 'username' in database or 'restore_username' in database or 'username' in connection_params else ()) + (('--no-owner',) if database['no_owner'] else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) @@ -247,7 +257,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) ) - extra_environment = make_extra_environment(database, restore=True) + extra_environment = make_extra_environment(database, restore=True, connection_params=connection_params) logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") if dry_run: From e15bec30e6036a23789c0ee1e7a058d530523758 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 13 Jun 2023 23:34:58 -0700 Subject: [PATCH 261/344] Mention some hang edge cases in database limitations (#710). --- docs/how-to/backup-your-databases.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 0a770d61..3aeea9b1 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -348,17 +348,19 @@ borgmatic's own configuration file. So include your configuration file in backups to avoid getting caught without a way to restore a database. 3. borgmatic does not currently support backing up or restoring multiple databases that share the exact same name on different hosts. -4. Because database hooks implicitly enable the `read_special` configuration -setting to support dump and restore streaming, you'll need to ensure that any -special files are excluded from backups (named pipes, block devices, -character devices, and sockets) to prevent hanging. Try a command like -`find /your/source/path -type b -or -type c -or -type p -or -type s` to find -such files. Common directories to exclude are `/dev` and `/run`, but that may -not be exhaustive. New in version +4. Because database hooks implicitly enable the `read_special` configuration, +any special files are excluded from backups (named pipes, block devices, +character devices, and sockets) to prevent hanging. Try a command like `find +/your/source/path -type b -or -type c -or -type p -or -type s` to find such +files. Common directories to exclude are `/dev` and `/run`, but that may not +be exhaustive. New in version 1.7.3 When database hooks are enabled, borgmatic automatically excludes -special files that may cause Borg to hang, so you no longer need to manually -exclude them. (This includes symlinks with special files as a destination.) You -can override/prevent this behavior by explicitly setting `read_special` to true. +special files (and symlinks to special files) that may cause Borg to hang, so +generally you no longer need to manually exclude them. There are potential +edge cases though in which applications on your system create new special files +*after* borgmatic constructs its exclude list, resulting in Borg hangs. If that +occurs, you can resort to the manual excludes described above. And to opt out +of the auto-exclude feature entirely, explicitly set `read_special` to true. ### Manual restoration From 67f4d43aece67321affa6ec45bd0fffd15d40748 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 15 Jun 2023 01:37:18 +0530 Subject: [PATCH 262/344] witten review --- borgmatic/actions/restore.py | 4 ---- borgmatic/config/schema.yaml | 6 +----- borgmatic/hooks/postgresql.py | 31 +++++++++++++++++-------------- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index a701aa0d..6d8eb4e6 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -79,10 +79,6 @@ def restore_single_database( f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}' ) - logger.info( - f'hostname port username password for database {database["name"]}' - ) - dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', hooks, diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 20fe3f97..87de0c64 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -816,11 +816,7 @@ properties: OWNER or SET SESSION AUTHORIZATION statements to set ownership of created schema elements. These statements will fail unless the initial - connection to the database is made by a superuser - (in which case they will execute as though wrapped - in SECURITY DEFINER functions). When --no-owner - is used, neither the ALTER OWNER nor SET SESSION - AUTHORIZATION statements will be emitted. + connection to the database is made by a superuser. format: type: string enum: ['plain', 'custom', 'directory', 'tar'] diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 89ade66f..d1c92f87 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -22,17 +22,20 @@ def make_dump_path(location_config): # pragma: no cover location_config.get('borgmatic_source_directory'), 'postgresql_databases' ) -def make_extra_environment(database, restore=False, connection_params=None): +def make_extra_environment(database, restore_connection_params=None): ''' Make the extra_environment dict from the given database configuration. + If restore connection params are given, this is for a restore operation. ''' extra = dict() if 'password' in database: extra['PGPASSWORD'] = database['password'] - if restore and 'restore_password' in database: - extra['PGPASSWORD'] = database['restore_password'] - if connection_params is not None and connection_params.get('password'): - extra['PGPASSWORD'] = connection_params['password'] + + try: + extra['PGPASSWORD'] = restore_connection_params.get('password') or database['restore_password'] + except (AttributeError, KeyError): + pass + extra['PGSSLMODE'] = database.get('ssl_mode', 'disable') if 'ssl_cert' in database: extra['PGSSLCERT'] = database['ssl_cert'] @@ -138,7 +141,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--username', database['username']) if 'username' in database else ()) - + (('--no-owner',) if database['no_owner'] else ()) + + (('--no-owner',) if database.get('no_owner', False) else ()) + (('--format', dump_format) if dump_format else ()) + (('--file', dump_filename) if dump_format == 'directory' else ()) + (tuple(database['options'].split(' ')) if 'options' in database else ()) @@ -230,9 +233,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, analyze_command = ( tuple(psql_command) + ('--no-password', '--no-psqlrc', '--quiet') - + (('--host', hostname) if 'hostname' in database or 'restore_hostname' in database or 'hostname' in connection_params else ()) - + (('--port', port) if 'port' in database or 'restore_port' in database or 'port' in connection_params else ()) - + (('--username', username) if 'username' in database or 'restore_username' in database or 'username' in connection_params else ()) + + (('--host', hostname) if hostname else ()) + + (('--port', port) if port else ()) + + (('--username', username) if username else ()) + (('--dbname', database['name']) if not all_databases else ()) + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ()) + ('--command', 'ANALYZE') @@ -244,10 +247,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, + ('--no-password',) + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean')) + (('--dbname', database['name']) if not all_databases else ()) - + (('--host', hostname) if 'hostname' in database or 'restore_hostname' in database or 'hostname' in connection_params else ()) - + (('--port', port) if 'port' in database or 'restore_port' in database or 'port' in connection_params else ()) - + (('--username', username) if 'username' in database or 'restore_username' in database or 'username' in connection_params else ()) - + (('--no-owner',) if database['no_owner'] else ()) + + (('--host', hostname) if hostname else ()) + + (('--port', port) if port else ()) + + (('--username', username) if username else ()) + + (('--no-owner',) if database.get('no_owner', False) else ()) + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) + (() if extract_process else (dump_filename,)) + tuple( @@ -257,7 +260,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) ) - extra_environment = make_extra_environment(database, restore=True, connection_params=connection_params) + extra_environment = make_extra_environment(database, restore_connection_params=connection_params) logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") if dry_run: From 205e5b152466f0315d3761a99002c7f4b3c2d8db Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 15 Jun 2023 01:47:46 +0530 Subject: [PATCH 263/344] mysql support --- borgmatic/hooks/mysql.py | 18 ++++++++++++------ borgmatic/hooks/postgresql.py | 5 +---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 793b78b0..b5218dc1 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -185,7 +185,7 @@ def make_database_dump_pattern( return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') -def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process): +def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params): ''' Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. @@ -199,15 +199,21 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, raise ValueError('The database configuration value is invalid') database = database_config[0] + + hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) + port = str(connection_params['port'] or database.get('restore_port', database.get('port'))) + username = connection_params['username'] or database.get('restore_username', database.get('username')) + password = connection_params['password'] or database.get('restore_password', database.get('password')) + restore_command = ( ('mysql', '--batch') + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) - + (('--host', database['hostname']) if 'hostname' in database else ()) - + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) + + (('--host', database['hostname']) if hostname else ()) + + (('--port', str(database['port'])) if port else ()) + + (('--protocol', 'tcp') if hostname or port else ()) + + (('--user', database['username']) if username else ()) ) - extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None + extra_environment = {'MYSQL_PWD': password} if password else None logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}") if dry_run: diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index d1c92f87..c90ae636 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -28,11 +28,9 @@ def make_extra_environment(database, restore_connection_params=None): If restore connection params are given, this is for a restore operation. ''' extra = dict() - if 'password' in database: - extra['PGPASSWORD'] = database['password'] try: - extra['PGPASSWORD'] = restore_connection_params.get('password') or database['restore_password'] + extra['PGPASSWORD'] = restore_connection_params.get('password') or database['restore_password'] or database['password'] except (AttributeError, KeyError): pass @@ -223,7 +221,6 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) port = str(connection_params['port'] or database.get('restore_port', database.get('port'))) username = connection_params['username'] or database.get('restore_username', database.get('username')) - password = connection_params['password'] or database.get('restore_password', database.get('password')) all_databases = bool(database['name'] == 'all') dump_filename = dump.make_database_dump_filename( From a9386b7a8789ace90b9bb6a4d0c6c29a4aee5d29 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 15 Jun 2023 02:18:24 +0530 Subject: [PATCH 264/344] add mongodb support, and sqlite restore path (config option only) --- borgmatic/config/schema.yaml | 52 ++++++++++++++++++++++++++++++++++++ borgmatic/hooks/mongodb.py | 27 +++++++++++-------- 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 87de0c64..77da6efe 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -952,16 +952,33 @@ properties: Database hostname to connect to. Defaults to connecting via local Unix socket. example: database.example.org + restore_hostname: + type: string + description: | + Database hostname to restore to. Defaults to + the "hostname" option. + example: database.example.org port: type: integer description: Port to connect to. Defaults to 3306. example: 3307 + restore_port: + type: integer + description: Port to restore to. Defaults to the + "port" option. + example: 5433 username: type: string description: | Username with which to connect to the database. Defaults to the username of the current user. example: dbuser + restore_username: + type: string + description: | + Username with which to restore the database. + Defaults to the "username" option. + example: dbuser password: type: string description: | @@ -970,6 +987,12 @@ properties: configured to trust the configured username without a password. example: trustsome1 + restore_password: + type: string + description: | + Password with which to connect to the restore + database. Defaults to the "password" option. + example: trustsome1 format: type: string enum: ['sql'] @@ -1047,6 +1070,12 @@ properties: read_special and one_file_system (see above) to support dump and restore streaming. example: /var/lib/sqlite/users.db + restore_path: + type: string + description: | + Path to the SQLite database file to restore to. + Defaults to the "path" option. + example: /var/lib/sqlite/users.db mongodb_databases: type: array items: @@ -1069,22 +1098,45 @@ properties: Database hostname to connect to. Defaults to connecting to localhost. example: database.example.org + restore_hostname: + type: string + description: | + Database hostname to restore to. Defaults to + the "hostname" option. + example: database.example.org port: type: integer description: Port to connect to. Defaults to 27017. example: 27018 + restore_port: + type: integer + description: Port to restore to. Defaults to the + "port" option. + example: 5433 username: type: string description: | Username with which to connect to the database. Skip it if no authentication is needed. example: dbuser + restore_username: + type: string + description: | + Username with which to restore the database. + Defaults to the "username" option. + example: dbuser password: type: string description: | Password with which to connect to the database. Skip it if no authentication is needed. example: trustsome1 + restore_password: + type: string + description: | + Password with which to connect to the restore + database. Defaults to the "password" option. + example: trustsome1 authentication_database: type: string description: | diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index 781e5f21..1cbd98f3 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -102,7 +102,7 @@ def make_database_dump_pattern( return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') -def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process): +def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params): ''' Restore the given MongoDB database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. @@ -122,7 +122,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, dump_filename = dump.make_database_dump_filename( make_dump_path(location_config), database['name'], database.get('hostname') ) - restore_command = build_restore_command(extract_process, database, dump_filename) + restore_command = build_restore_command(extract_process, database, dump_filename, connection_params) logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}") if dry_run: @@ -138,10 +138,15 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) -def build_restore_command(extract_process, database, dump_filename): +def build_restore_command(extract_process, database, dump_filename, connection_params): ''' Return the mongorestore command from a single database configuration. ''' + hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) + port = str(connection_params['port'] or database.get('restore_port', database.get('port'))) + username = connection_params['username'] or database.get('restore_username', database.get('username')) + password = connection_params['password'] or database.get('restore_password', database.get('password')) + command = ['mongorestore'] if extract_process: command.append('--archive') @@ -149,14 +154,14 @@ def build_restore_command(extract_process, database, dump_filename): command.extend(('--dir', dump_filename)) if database['name'] != 'all': command.extend(('--drop', '--db', database['name'])) - if 'hostname' in database: - command.extend(('--host', database['hostname'])) - if 'port' in database: - command.extend(('--port', str(database['port']))) - if 'username' in database: - command.extend(('--username', database['username'])) - if 'password' in database: - command.extend(('--password', database['password'])) + if hostname: + command.extend(('--host', hostname)) + if port: + command.extend(('--port', str(port))) + if username: + command.extend(('--username', username)) + if password: + command.extend(('--password', password)) if 'authentication_database' in database: command.extend(('--authenticationDatabase', database['authentication_database'])) if 'restore_options' in database: From 1d7c7eaaa7ec0ab67add6e95c7a95a14ec1498ca Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 14 Jun 2023 14:57:57 -0700 Subject: [PATCH 265/344] Add sample systemd user serivce for running borgmatic as a non-root user (#669). --- NEWS | 1 + sample/systemd/borgmatic-user.service | 15 +++++++++++++++ sample/systemd/borgmatic-user.timer | 1 + 3 files changed, 17 insertions(+) create mode 100644 sample/systemd/borgmatic-user.service create mode 120000 sample/systemd/borgmatic-user.timer diff --git a/NEWS b/NEWS index 15344100..a5a1014e 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.7.15.dev0 * #697: Extract borgmatic configuration from backup via "bootstrap" action—even when borgmatic has no configuration yet! + * #669: Add sample systemd user serivce for running borgmatic as a non-root user. 1.7.14 * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file, diff --git a/sample/systemd/borgmatic-user.service b/sample/systemd/borgmatic-user.service new file mode 100644 index 00000000..e5b4280b --- /dev/null +++ b/sample/systemd/borgmatic-user.service @@ -0,0 +1,15 @@ +[Unit] +Description=borgmatic backup +Wants=network-online.target +After=network-online.target +ConditionACPower=true + +[Service] +Type=oneshot +Restart=no +# Prevent rate limiting of borgmatic log events. If you are using an older version of systemd that +# doesn't support this (pre-240 or so), you may have to remove this option. +LogRateLimitIntervalSec=0 +# Delay start to prevent backups running during boot. +ExecStartPre=sleep 1m +ExecStart=/root/.local/bin/borgmatic --verbosity -2 --syslog-verbosity 1 diff --git a/sample/systemd/borgmatic-user.timer b/sample/systemd/borgmatic-user.timer new file mode 120000 index 00000000..19e3f350 --- /dev/null +++ b/sample/systemd/borgmatic-user.timer @@ -0,0 +1 @@ +borgmatic.timer \ No newline at end of file From b7423c488e9df2174cbbbbaebdf5a9c6f319b9c6 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 15 Jun 2023 22:54:06 +0530 Subject: [PATCH 266/344] refactor password assignment logic --- borgmatic/actions/restore.py | 2 +- borgmatic/hooks/postgresql.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 6d8eb4e6..532c5876 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -72,7 +72,7 @@ def restore_single_database( ): # pragma: no cover ''' Given (among other things) an archive name, a database hook name, the hostname, - port, username and password as connection params, and a configured database + port, username and password as connection params, and a configured database configuration dict, restore that database from the archive. ''' logger.info( diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index c90ae636..86720281 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -30,7 +30,10 @@ def make_extra_environment(database, restore_connection_params=None): extra = dict() try: - extra['PGPASSWORD'] = restore_connection_params.get('password') or database['restore_password'] or database['password'] + if restore_connection_params: + extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get('restore_password', database['password']) + else: + extra['PGPASSWORD'] = database['password'] except (AttributeError, KeyError): pass From 62b6f1329923020da8ffdbe0dc6109f5358189bb Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 15 Jun 2023 23:02:09 +0530 Subject: [PATCH 267/344] add restore-path support for sqlite --- borgmatic/actions/restore.py | 1 + borgmatic/hooks/sqlite.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 532c5876..156f3763 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -316,6 +316,7 @@ def run_restore( 'port': restore_arguments.port, 'username': restore_arguments.username, 'password': restore_arguments.password, + 'restore_path': restore_arguments.restore_path, } if not found_database: diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py index d9f105d8..4e4e0be5 100644 --- a/borgmatic/hooks/sqlite.py +++ b/borgmatic/hooks/sqlite.py @@ -85,7 +85,7 @@ def make_database_dump_pattern( return dump.make_database_dump_filename(make_dump_path(location_config), name) -def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process): +def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params): ''' Restore the given SQLite3 database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. @@ -98,7 +98,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, if len(database_config) != 1: raise ValueError('The database configuration value is invalid') - database_path = database_config[0]['path'] + database_path = connection_params['restore_path'] or database_config[0].get('restore_path', database_config[0].get('path')) logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}') if dry_run: From 82d851d8911b0e77e6705d2196908e343b40383d Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 15 Jun 2023 23:05:53 +0530 Subject: [PATCH 268/344] add argument for restore path --- borgmatic/commands/arguments.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 02f2fc3e..61db1980 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -735,6 +735,10 @@ def make_parsers(): '--password', help='Password with which to connect to the restore database. Defaults to the "restore_password" option in borgmatic\'s configuration', ) + restore_group.add_argument( + '--restore-path', + help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration', + ) restore_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' ) From bbc7f0596c13b79e6930ffa1a2b4381e8d2df766 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 15 Jun 2023 10:55:31 -0700 Subject: [PATCH 269/344] Fix Bash completion for sub-actions like "borgmatic config bootstrap" (#697 follow-on work). --- NEWS | 2 +- borgmatic/commands/borgmatic.py | 7 +- borgmatic/commands/completion/__init__.py | 0 borgmatic/commands/completion/actions.py | 44 ++++++++++++ borgmatic/commands/completion/bash.py | 62 ++++++++++++++++ .../{completion.py => completion/fish.py} | 72 ++----------------- .../commands/completion/__init__.py | 0 .../commands/completion/test_actions.py | 20 ++++++ .../commands/completion/test_bash.py | 5 ++ .../commands/completion/test_fish.py | 5 ++ tests/integration/commands/test_completion.py | 9 --- tests/unit/commands/completion/__init__.py | 0 .../unit/commands/completion/test_actions.py | 7 ++ tests/unit/commands/completion/test_bash.py | 17 +++++ .../test_fish.py} | 2 +- 15 files changed, 170 insertions(+), 82 deletions(-) create mode 100644 borgmatic/commands/completion/__init__.py create mode 100644 borgmatic/commands/completion/actions.py create mode 100644 borgmatic/commands/completion/bash.py rename borgmatic/commands/{completion.py => completion/fish.py} (72%) create mode 100644 tests/integration/commands/completion/__init__.py create mode 100644 tests/integration/commands/completion/test_actions.py create mode 100644 tests/integration/commands/completion/test_bash.py create mode 100644 tests/integration/commands/completion/test_fish.py delete mode 100644 tests/integration/commands/test_completion.py create mode 100644 tests/unit/commands/completion/__init__.py create mode 100644 tests/unit/commands/completion/test_actions.py create mode 100644 tests/unit/commands/completion/test_bash.py rename tests/unit/commands/{test_completions.py => completion/test_fish.py} (98%) diff --git a/NEWS b/NEWS index a5a1014e..8c5480fe 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,7 @@ 1.7.15.dev0 * #697: Extract borgmatic configuration from backup via "bootstrap" action—even when borgmatic has no configuration yet! - * #669: Add sample systemd user serivce for running borgmatic as a non-root user. + * #669: Add sample systemd user service for running borgmatic as a non-root user. 1.7.14 * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file, diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 6523ebbc..79aae78b 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -31,7 +31,8 @@ import borgmatic.actions.restore import borgmatic.actions.rinfo import borgmatic.actions.rlist import borgmatic.actions.transfer -import borgmatic.commands.completion +import borgmatic.commands.completion.bash +import borgmatic.commands.completion.fish from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments @@ -751,10 +752,10 @@ def main(): # pragma: no cover print(importlib_metadata.version('borgmatic')) sys.exit(0) if global_arguments.bash_completion: - print(borgmatic.commands.completion.bash_completion()) + print(borgmatic.commands.completion.bash.bash_completion()) sys.exit(0) if global_arguments.fish_completion: - print(borgmatic.commands.completion.fish_completion()) + print(borgmatic.commands.completion.fish.fish_completion()) sys.exit(0) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) diff --git a/borgmatic/commands/completion/__init__.py b/borgmatic/commands/completion/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/borgmatic/commands/completion/actions.py b/borgmatic/commands/completion/actions.py new file mode 100644 index 00000000..903670da --- /dev/null +++ b/borgmatic/commands/completion/actions.py @@ -0,0 +1,44 @@ +import argparse + + +def upgrade_message(language: str, upgrade_command: str, completion_file: str): + return f''' +Your {language} completions script is from a different version of borgmatic than is +currently installed. Please upgrade your script so your completions match the +command-line flags in your installed borgmatic! Try this to upgrade: + + {upgrade_command} + source {completion_file} +''' + + +def available_actions(subparsers, current_action=None): + ''' + Given subparsers as an argparse._SubParsersAction instance and a current action name (if + any), return the actions names that can follow the current action on a command-line. + + This takes into account which sub-actions that the current action supports. For instance, if + "bootstrap" is a sub-action for "config", then "bootstrap" should be able to follow a current + action of "config" but not "list". + ''' + # Make a map from action name to the names of contained sub-actions. + actions_to_subactions = { + action: tuple( + subaction_name + for subaction in subparser._actions + if isinstance(subaction, argparse._SubParsersAction) + for subaction_name in subaction.choices.keys() + ) + for action, subparser in subparsers.choices.items() + } + + current_subactions = actions_to_subactions.get(current_action) + + if current_subactions: + return current_subactions + + all_subactions = set( + subaction for subactions in actions_to_subactions.values() for subaction in subactions + ) + + return tuple(action for action in subparsers.choices.keys() if action not in all_subactions) diff --git a/borgmatic/commands/completion/bash.py b/borgmatic/commands/completion/bash.py new file mode 100644 index 00000000..d20eca42 --- /dev/null +++ b/borgmatic/commands/completion/bash.py @@ -0,0 +1,62 @@ +import borgmatic.commands.arguments +import borgmatic.commands.completion.actions + + +def parser_flags(parser): + ''' + Given an argparse.ArgumentParser instance, return its argument flags in a space-separated + string. + ''' + return ' '.join(option for action in parser._actions for option in action.option_strings) + + +def bash_completion(): + ''' + Return a bash completion script for the borgmatic command. Produce this by introspecting + borgmatic's command-line argument parsers. + ''' + top_level_parser, subparsers = borgmatic.commands.arguments.make_parsers() + global_flags = parser_flags(top_level_parser) + + # Avert your eyes. + return '\n'.join( + ( + 'check_version() {', + ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', + ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', + ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' + f''' then cat << EOF\n{borgmatic.commands.completion.actions.upgrade_message( + 'bash', + 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', + '$BASH_SOURCE', + )}\nEOF''', + ' fi', + '}', + 'complete_borgmatic() {', + ) + + tuple( + ''' if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then + COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}")) + return 0 + fi''' + % ( + action, + parser_flags(subparser), + ' '.join( + borgmatic.commands.completion.actions.available_actions(subparsers, action) + ), + global_flags, + ) + for action, subparser in reversed(subparsers.choices.items()) + ) + + ( + ' COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))' # noqa: FS003 + % ( + ' '.join(borgmatic.commands.completion.actions.available_actions(subparsers)), + global_flags, + ), + ' (check_version &)', + '}', + '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic', + ) + ) diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion/fish.py similarity index 72% rename from borgmatic/commands/completion.py rename to borgmatic/commands/completion/fish.py index 4b2f17f3..306de195 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion/fish.py @@ -2,72 +2,8 @@ import shlex from argparse import Action from textwrap import dedent -from borgmatic.commands import arguments - - -def upgrade_message(language: str, upgrade_command: str, completion_file: str): - return f''' -Your {language} completions script is from a different version of borgmatic than is -currently installed. Please upgrade your script so your completions match the -command-line flags in your installed borgmatic! Try this to upgrade: - - {upgrade_command} - source {completion_file} -''' - - -def parser_flags(parser): - ''' - Given an argparse.ArgumentParser instance, return its argument flags in a space-separated - string. - ''' - return ' '.join(option for action in parser._actions for option in action.option_strings) - - -def bash_completion(): - ''' - Return a bash completion script for the borgmatic command. Produce this by introspecting - borgmatic's command-line argument parsers. - ''' - top_level_parser, subparsers = arguments.make_parsers() - global_flags = parser_flags(top_level_parser) - actions = ' '.join(subparsers.choices.keys()) - - # Avert your eyes. - return '\n'.join( - ( - 'check_version() {', - ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', - ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', - ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - f''' then cat << EOF\n{upgrade_message( - 'bash', - 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', - '$BASH_SOURCE', - )}\nEOF''', - ' fi', - '}', - 'complete_borgmatic() {', - ) - + tuple( - ''' if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then - COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}")) - return 0 - fi''' - % (action, parser_flags(subparser), actions, global_flags) - for action, subparser in subparsers.choices.items() - ) - + ( - ' COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))' # noqa: FS003 - % (actions, global_flags), - ' (check_version &)', - '}', - '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic', - ) - ) - - -# fish section +import borgmatic.commands.arguments +import borgmatic.commands.completion.actions def has_file_options(action: Action): @@ -155,7 +91,7 @@ def fish_completion(): Return a fish completion script for the borgmatic command. Produce this by introspecting borgmatic's command-line argument parsers. ''' - top_level_parser, subparsers = arguments.make_parsers() + top_level_parser, subparsers = borgmatic.commands.arguments.make_parsers() all_subparsers = ' '.join(action for action in subparsers.choices.keys()) @@ -182,7 +118,7 @@ def fish_completion(): set this_script (cat $this_filename 2> /dev/null) set installed_script (borgmatic --fish-completion 2> /dev/null) if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] - echo "{upgrade_message( + echo "{borgmatic.commands.completion.actions.upgrade_message( 'fish', 'borgmatic --fish-completion | sudo tee $this_filename', '$this_filename', diff --git a/tests/integration/commands/completion/__init__.py b/tests/integration/commands/completion/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/commands/completion/test_actions.py b/tests/integration/commands/completion/test_actions.py new file mode 100644 index 00000000..1a8d69e6 --- /dev/null +++ b/tests/integration/commands/completion/test_actions.py @@ -0,0 +1,20 @@ +import borgmatic.commands.arguments +from borgmatic.commands.completion import actions as module + + +def test_available_actions_uses_only_subactions_for_action_with_subactions(): + unused_top_level_parser, subparsers = borgmatic.commands.arguments.make_parsers() + + actions = module.available_actions(subparsers, 'config') + + assert 'bootstrap' in actions + assert 'list' not in actions + + +def test_available_actions_omits_subactions_for_action_without_subactions(): + unused_top_level_parser, subparsers = borgmatic.commands.arguments.make_parsers() + + actions = module.available_actions(subparsers, 'list') + + assert 'bootstrap' not in actions + assert 'config' in actions diff --git a/tests/integration/commands/completion/test_bash.py b/tests/integration/commands/completion/test_bash.py new file mode 100644 index 00000000..ac3cff63 --- /dev/null +++ b/tests/integration/commands/completion/test_bash.py @@ -0,0 +1,5 @@ +from borgmatic.commands.completion import bash as module + + +def test_bash_completion_does_not_raise(): + assert module.bash_completion() diff --git a/tests/integration/commands/completion/test_fish.py b/tests/integration/commands/completion/test_fish.py new file mode 100644 index 00000000..3b7d3dd8 --- /dev/null +++ b/tests/integration/commands/completion/test_fish.py @@ -0,0 +1,5 @@ +from borgmatic.commands.completion import fish as module + + +def test_fish_completion_does_not_raise(): + assert module.fish_completion() diff --git a/tests/integration/commands/test_completion.py b/tests/integration/commands/test_completion.py deleted file mode 100644 index 9a118abf..00000000 --- a/tests/integration/commands/test_completion.py +++ /dev/null @@ -1,9 +0,0 @@ -from borgmatic.commands import completion as module - - -def test_bash_completion_does_not_raise(): - assert module.bash_completion() - - -def test_fish_completion_does_not_raise(): - assert module.fish_completion() diff --git a/tests/unit/commands/completion/__init__.py b/tests/unit/commands/completion/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/commands/completion/test_actions.py b/tests/unit/commands/completion/test_actions.py new file mode 100644 index 00000000..75f3dead --- /dev/null +++ b/tests/unit/commands/completion/test_actions.py @@ -0,0 +1,7 @@ +from borgmatic.commands.completion import actions as module + + +def test_upgrade_message_does_not_raise(): + module.upgrade_message( + language='English', upgrade_command='read a lot', completion_file='your brain' + ) diff --git a/tests/unit/commands/completion/test_bash.py b/tests/unit/commands/completion/test_bash.py new file mode 100644 index 00000000..3f4265bd --- /dev/null +++ b/tests/unit/commands/completion/test_bash.py @@ -0,0 +1,17 @@ +from flexmock import flexmock + +from borgmatic.commands.completion import bash as module + + +def test_parser_flags_flattens_and_joins_flags(): + assert ( + module.parser_flags( + flexmock( + _actions=[ + flexmock(option_strings=['--foo', '--bar']), + flexmock(option_strings=['--baz']), + ] + ) + ) + == '--foo --bar --baz' + ) diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/completion/test_fish.py similarity index 98% rename from tests/unit/commands/test_completions.py rename to tests/unit/commands/completion/test_fish.py index 12829d5f..35d53ed8 100644 --- a/tests/unit/commands/test_completions.py +++ b/tests/unit/commands/completion/test_fish.py @@ -5,7 +5,7 @@ from typing import Tuple import pytest from flexmock import flexmock -from borgmatic.commands import completion as module +from borgmatic.commands.completion import fish as module OptionType = namedtuple('OptionType', ['file', 'choice', 'unknown_required']) TestCase = Tuple[Action, OptionType] From 8389851f2f907164713a621240b4e0f6e2e66663 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Thu, 15 Jun 2023 23:34:50 +0530 Subject: [PATCH 270/344] fix bug where port becomes truthy when none is converted to str --- borgmatic/hooks/mongodb.py | 2 +- borgmatic/hooks/mysql.py | 2 +- borgmatic/hooks/postgresql.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index 1cbd98f3..f8432b1f 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -143,7 +143,7 @@ def build_restore_command(extract_process, database, dump_filename, connection_p Return the mongorestore command from a single database configuration. ''' hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) - port = str(connection_params['port'] or database.get('restore_port', database.get('port'))) + port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) username = connection_params['username'] or database.get('restore_username', database.get('username')) password = connection_params['password'] or database.get('restore_password', database.get('password')) diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index b5218dc1..f64f16f2 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -201,7 +201,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, database = database_config[0] hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) - port = str(connection_params['port'] or database.get('restore_port', database.get('port'))) + port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) username = connection_params['username'] or database.get('restore_username', database.get('username')) password = connection_params['password'] or database.get('restore_password', database.get('password')) diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 86720281..08b0019c 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -222,7 +222,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, database = database_config[0] hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) - port = str(connection_params['port'] or database.get('restore_port', database.get('port'))) + port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) username = connection_params['username'] or database.get('restore_username', database.get('username')) all_databases = bool(database['name'] == 'all') From 9152fed249322f06343bda93984bac262d2012db Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 15 Jun 2023 14:55:57 -0700 Subject: [PATCH 271/344] Add a documentation troubleshooting note for MySQL/MariaDB authentication errors (#399). --- NEWS | 1 + docs/how-to/backup-your-databases.md | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/NEWS b/NEWS index 8c5480fe..6f230e43 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 1.7.15.dev0 + * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors. * #697: Extract borgmatic configuration from backup via "bootstrap" action—even when borgmatic has no configuration yet! * #669: Add sample systemd user service for running borgmatic as a non-root user. diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 3aeea9b1..fea3f350 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -410,6 +410,12 @@ authenticated. For instance, with PostgreSQL, check your [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) file for that configuration. +Additionally, MySQL/MariaDB may be picking up some of your credentials from a +defaults file like `~/.my.cnf`. If that's the case, then it's possible +MySQL/MariaDB ends up using, say, a username from borgmatic's configuration +and a password from `~/.my.cnf`. This may result in authentication errors if +this combination of credentials is not what you intend. + ### MySQL table lock errors From c294e78715c139282ed15255166f3c97f04c52be Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 15 Jun 2023 21:45:43 -0700 Subject: [PATCH 272/344] Use absolute paths when storing configuration files in an archive for later bootstrapping (#697). --- borgmatic/actions/config/bootstrap.py | 35 ++++++++++++------------- borgmatic/commands/borgmatic.py | 37 +++++++++++++-------------- borgmatic/config/collect.py | 10 ++++---- tests/unit/config/test_collect.py | 31 +++++++++++----------- 4 files changed, 55 insertions(+), 58 deletions(-) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 7ece946f..2f5acbd4 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -81,26 +81,25 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): bootstrap_arguments, global_arguments, local_borg_version ) - for config_path in manifest_config_paths: - logger.info(f'Bootstrapping config path {config_path}') + logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}") - borgmatic.borg.extract.extract_archive( - global_arguments.dry_run, + borgmatic.borg.extract.extract_archive( + global_arguments.dry_run, + bootstrap_arguments.repository, + borgmatic.borg.rlist.resolve_archive_name( bootstrap_arguments.repository, - borgmatic.borg.rlist.resolve_archive_name( - bootstrap_arguments.repository, - bootstrap_arguments.archive, - {}, - local_borg_version, - global_arguments, - ), - [config_path], - {}, + bootstrap_arguments.archive, {}, local_borg_version, global_arguments, - extract_to_stdout=False, - destination_path=bootstrap_arguments.destination, - strip_components=bootstrap_arguments.strip_components, - progress=bootstrap_arguments.progress, - ) + ), + [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths], + {}, + {}, + local_borg_version, + global_arguments, + extract_to_stdout=False, + destination_path=bootstrap_arguments.destination, + strip_components=bootstrap_arguments.strip_components, + progress=bootstrap_arguments.progress, + ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 79aae78b..c670291f 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -612,26 +612,8 @@ def collect_configuration_run_summary_logs(configs, arguments): As a side effect of running through these configuration files, output their JSON results, if any, to stdout. ''' - # Run cross-file validation checks. - repository = None - - for action_name, action_arguments in arguments.items(): - if hasattr(action_arguments, 'repository'): - repository = getattr(action_arguments, 'repository') - break - - try: - if 'extract' in arguments or 'mount' in arguments: - validate.guard_single_repository_selected(repository, configs) - - if 'bootstrap' not in arguments: - validate.guard_configuration_contains_repository(repository, configs) - except ValueError as error: - yield from log_error_records(str(error)) - return - if 'bootstrap' in arguments: - # no configuration file is needed for bootstrap + # No configuration file is needed for bootstrap. local_borg_version = borg_version.local_borg_version({}, 'borg') try: borgmatic.actions.config.bootstrap.run_bootstrap( @@ -653,6 +635,23 @@ def collect_configuration_run_summary_logs(configs, arguments): return + # Run cross-file validation checks. + repository = None + + for action_name, action_arguments in arguments.items(): + if hasattr(action_arguments, 'repository'): + repository = getattr(action_arguments, 'repository') + break + + try: + if 'extract' in arguments or 'mount' in arguments: + validate.guard_single_repository_selected(repository, configs) + + validate.guard_configuration_contains_repository(repository, configs) + except ValueError as error: + yield from log_error_records(str(error)) + return + if not configs: yield from log_error_records( f"{' '.join(arguments['global'].config_paths)}: No valid configuration files found", diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py index bd38fee2..80c6c5c0 100644 --- a/borgmatic/config/collect.py +++ b/borgmatic/config/collect.py @@ -24,9 +24,9 @@ def get_default_config_paths(expand_home=True): def collect_config_filenames(config_paths): ''' Given a sequence of config paths, both filenames and directories, resolve that to an iterable - of files. Accomplish this by listing any given directories looking for contained config files - (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories within the given - directories are ignored. + of absolute files. Accomplish this by listing any given directories looking for contained config + files (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories + within the given directories are ignored. Return paths even if they don't exist on disk, so the user can find out about missing configuration paths. However, skip a default config path if it's missing, so the user doesn't @@ -41,7 +41,7 @@ def collect_config_filenames(config_paths): continue if not os.path.isdir(path) or not exists: - yield path + yield os.path.abspath(path) continue if not os.access(path, os.R_OK): @@ -51,4 +51,4 @@ def collect_config_filenames(config_paths): full_filename = os.path.join(path, filename) matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml') if matching_filetype and not os.path.isdir(full_filename): - yield full_filename + yield os.path.abspath(full_filename) diff --git a/tests/unit/config/test_collect.py b/tests/unit/config/test_collect.py index 5bab1ae0..d20c006f 100644 --- a/tests/unit/config/test_collect.py +++ b/tests/unit/config/test_collect.py @@ -29,15 +29,6 @@ def test_get_default_config_paths_does_not_expand_home_when_false(): assert '$HOME/.config/borgmatic/config.yaml' in config_paths -def test_collect_config_filenames_collects_given_files(): - config_paths = ('config.yaml', 'other.yaml') - flexmock(module.os.path).should_receive('isdir').and_return(False) - - config_filenames = tuple(module.collect_config_filenames(config_paths)) - - assert config_filenames == config_paths - - def test_collect_config_filenames_collects_yml_file_endings(): config_paths = ('config.yaml', '/etc/borgmatic.d') mock_path = flexmock(module.os.path) @@ -45,13 +36,14 @@ def test_collect_config_filenames_collects_yml_file_endings(): mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yml').and_return(False) + mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path)) flexmock(module.os).should_receive('access').and_return(True) flexmock(module.os).should_receive('listdir') flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['foo.yml']) config_filenames = tuple(module.collect_config_filenames(config_paths)) - assert config_filenames == ('config.yaml', '/etc/borgmatic.d/foo.yml') + assert config_filenames == ('/config.yaml', '/etc/borgmatic.d/foo.yml') def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories(): @@ -63,6 +55,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False) + mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path)) flexmock(module.os).should_receive('access').and_return(True) flexmock(module.os).should_receive('listdir') flexmock(sys.modules['builtins']).should_receive('sorted').and_return( @@ -72,7 +65,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno config_filenames = tuple(module.collect_config_filenames(config_paths)) assert config_filenames == ( - 'config.yaml', + '/config.yaml', '/etc/borgmatic.d/foo.yaml', '/etc/borgmatic.d/baz.yaml', ) @@ -86,6 +79,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False) + mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path)) flexmock(module.os).should_receive('access').and_return(True) flexmock(module.os).should_receive('listdir') flexmock(sys.modules['builtins']).should_receive('sorted').and_return( @@ -103,13 +97,14 @@ def test_collect_config_filenames_skips_permission_denied_directories(): mock_path.should_receive('exists').and_return(True) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) + mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path)) flexmock(module.os).should_receive('access').and_return(False) flexmock(module.os).should_receive('listdir') flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['config.yaml']) config_filenames = tuple(module.collect_config_filenames(config_paths)) - assert config_filenames == ('config.yaml',) + assert config_filenames == ('/config.yaml',) def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist(): @@ -119,10 +114,11 @@ def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does mock_path.should_receive('exists').with_args('/etc/borgmatic/config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic/config.yaml').and_return(True) + mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path)) config_filenames = tuple(module.collect_config_filenames(config_paths)) - assert config_filenames == ('config.yaml',) + assert config_filenames == ('/config.yaml',) def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist(): @@ -132,10 +128,11 @@ def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) + mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path)) config_filenames = tuple(module.collect_config_filenames(config_paths)) - assert config_filenames == ('config.yaml',) + assert config_filenames == ('/config.yaml',) def test_collect_config_filenames_skips_non_canonical_etc_borgmatic_dot_d_if_it_does_not_exist(): @@ -145,10 +142,11 @@ def test_collect_config_filenames_skips_non_canonical_etc_borgmatic_dot_d_if_it_ mock_path.should_receive('exists').with_args('/etc/../etc/borgmatic.d').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/../etc/borgmatic.d').and_return(True) + mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path)) config_filenames = tuple(module.collect_config_filenames(config_paths)) - assert config_filenames == ('config.yaml',) + assert config_filenames == ('/config.yaml',) def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist(): @@ -158,7 +156,8 @@ def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist( mock_path.should_receive('exists').with_args('/my/directory').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/my/directory').and_return(True) + mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path)) config_filenames = tuple(module.collect_config_filenames(config_paths)) - assert config_filenames == config_paths + assert config_filenames == ('/config.yaml', '/my/directory') From 89602d1614b80bd14e6ab314bd6f84ba49c352bf Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Fri, 16 Jun 2023 15:14:00 +0530 Subject: [PATCH 273/344] pass all existing tests (and formatting) --- borgmatic/actions/restore.py | 16 ++-- borgmatic/commands/arguments.py | 3 +- borgmatic/config/schema.yaml | 5 +- borgmatic/hooks/mongodb.py | 20 +++-- borgmatic/hooks/mysql.py | 16 +++- borgmatic/hooks/postgresql.py | 23 +++-- borgmatic/hooks/sqlite.py | 8 +- tests/unit/actions/test_restore.py | 48 +++++++++- tests/unit/hooks/test_mongodb.py | 108 +++++++++++++++++++++-- tests/unit/hooks/test_mysql.py | 72 +++++++++++++-- tests/unit/hooks/test_postgresql.py | 132 +++++++++++++++++++++++++--- tests/unit/hooks/test_sqlite.py | 21 ++++- 12 files changed, 411 insertions(+), 61 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 156f3763..d44a2cac 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -304,6 +304,13 @@ def run_restore( restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names) found_names = set() remaining_restore_names = {} + connection_params = { + 'hostname': restore_arguments.hostname, + 'port': restore_arguments.port, + 'username': restore_arguments.username, + 'password': restore_arguments.password, + 'restore_path': restore_arguments.restore_path, + } for hook_name, database_names in restore_names.items(): for database_name in database_names: @@ -311,14 +318,6 @@ def run_restore( hooks, archive_database_names, hook_name, database_name ) - connection_params = { - 'hostname': restore_arguments.hostname, - 'port': restore_arguments.port, - 'username': restore_arguments.username, - 'password': restore_arguments.password, - 'restore_path': restore_arguments.restore_path, - } - if not found_database: remaining_restore_names.setdefault(found_hook_name or hook_name, []).append( database_name @@ -368,6 +367,7 @@ def run_restore( archive_name, found_hook_name or hook_name, dict(database, **{'schemas': restore_arguments.schemas}), + connection_params, ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 61db1980..241e6efd 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -725,7 +725,8 @@ def make_parsers(): help='Database hostname to restore to. Defaults to the "restore_hostname" option in borgmatic\'s configuration', ) restore_group.add_argument( - '--port', help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration' + '--port', + help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration', ) restore_group.add_argument( '--username', diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 77da6efe..7d079ff0 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -816,7 +816,8 @@ properties: OWNER or SET SESSION AUTHORIZATION statements to set ownership of created schema elements. These statements will fail unless the initial - connection to the database is made by a superuser. + connection to the database is made by a + superuser. format: type: string enum: ['plain', 'custom', 'directory', 'tar'] @@ -1103,7 +1104,7 @@ properties: description: | Database hostname to restore to. Defaults to the "hostname" option. - example: database.example.org + example: database.example.org port: type: integer description: Port to connect to. Defaults to 27017. diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index f8432b1f..f8899268 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -102,7 +102,9 @@ def make_database_dump_pattern( return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') -def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params): +def restore_database_dump( + database_config, log_prefix, location_config, dry_run, extract_process, connection_params +): ''' Restore the given MongoDB database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. @@ -122,7 +124,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, dump_filename = dump.make_database_dump_filename( make_dump_path(location_config), database['name'], database.get('hostname') ) - restore_command = build_restore_command(extract_process, database, dump_filename, connection_params) + restore_command = build_restore_command( + extract_process, database, dump_filename, connection_params + ) logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}") if dry_run: @@ -142,10 +146,16 @@ def build_restore_command(extract_process, database, dump_filename, connection_p ''' Return the mongorestore command from a single database configuration. ''' - hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) + hostname = connection_params['hostname'] or database.get( + 'restore_hostname', database.get('hostname') + ) port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) - username = connection_params['username'] or database.get('restore_username', database.get('username')) - password = connection_params['password'] or database.get('restore_password', database.get('password')) + username = connection_params['username'] or database.get( + 'restore_username', database.get('username') + ) + password = connection_params['password'] or database.get( + 'restore_password', database.get('password') + ) command = ['mongorestore'] if extract_process: diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index f64f16f2..22eef61e 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -185,7 +185,9 @@ def make_database_dump_pattern( return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') -def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params): +def restore_database_dump( + database_config, log_prefix, location_config, dry_run, extract_process, connection_params +): ''' Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. @@ -200,10 +202,16 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, database = database_config[0] - hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) + hostname = connection_params['hostname'] or database.get( + 'restore_hostname', database.get('hostname') + ) port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) - username = connection_params['username'] or database.get('restore_username', database.get('username')) - password = connection_params['password'] or database.get('restore_password', database.get('password')) + username = connection_params['username'] or database.get( + 'restore_username', database.get('username') + ) + password = connection_params['password'] or database.get( + 'restore_password', database.get('password') + ) restore_command = ( ('mysql', '--batch') diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 08b0019c..ecb5f3c3 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -22,6 +22,7 @@ def make_dump_path(location_config): # pragma: no cover location_config.get('borgmatic_source_directory'), 'postgresql_databases' ) + def make_extra_environment(database, restore_connection_params=None): ''' Make the extra_environment dict from the given database configuration. @@ -31,12 +32,14 @@ def make_extra_environment(database, restore_connection_params=None): try: if restore_connection_params: - extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get('restore_password', database['password']) + extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get( + 'restore_password', database['password'] + ) else: extra['PGPASSWORD'] = database['password'] except (AttributeError, KeyError): pass - + extra['PGSSLMODE'] = database.get('ssl_mode', 'disable') if 'ssl_cert' in database: extra['PGSSLCERT'] = database['ssl_cert'] @@ -200,7 +203,9 @@ def make_database_dump_pattern( return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') -def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params): +def restore_database_dump( + database_config, log_prefix, location_config, dry_run, extract_process, connection_params +): ''' Restore the given PostgreSQL database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. @@ -221,9 +226,13 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, database = database_config[0] - hostname = connection_params['hostname'] or database.get('restore_hostname', database.get('hostname')) + hostname = connection_params['hostname'] or database.get( + 'restore_hostname', database.get('hostname') + ) port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) - username = connection_params['username'] or database.get('restore_username', database.get('username')) + username = connection_params['username'] or database.get( + 'restore_username', database.get('username') + ) all_databases = bool(database['name'] == 'all') dump_filename = dump.make_database_dump_filename( @@ -260,7 +269,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, ) ) - extra_environment = make_extra_environment(database, restore_connection_params=connection_params) + extra_environment = make_extra_environment( + database, restore_connection_params=connection_params + ) logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}") if dry_run: diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py index 4e4e0be5..21b1455a 100644 --- a/borgmatic/hooks/sqlite.py +++ b/borgmatic/hooks/sqlite.py @@ -85,7 +85,9 @@ def make_database_dump_pattern( return dump.make_database_dump_filename(make_dump_path(location_config), name) -def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process, connection_params): +def restore_database_dump( + database_config, log_prefix, location_config, dry_run, extract_process, connection_params +): ''' Restore the given SQLite3 database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. @@ -98,7 +100,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run, if len(database_config) != 1: raise ValueError('The database configuration value is invalid') - database_path = connection_params['restore_path'] or database_config[0].get('restore_path', database_config[0].get('path')) + database_path = connection_params['restore_path'] or database_config[0].get( + 'restore_path', database_config[0].get('path') + ) logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}') if dry_run: diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 4bad6f82..4e19964f 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -241,6 +241,7 @@ def test_run_restore_restores_each_database(): archive_name=object, hook_name='postgresql_databases', database={'name': 'foo', 'schemas': None}, + connection_params=object, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -254,6 +255,7 @@ def test_run_restore_restores_each_database(): archive_name=object, hook_name='postgresql_databases', database={'name': 'bar', 'schemas': None}, + connection_params=object, ).once() flexmock(module).should_receive('ensure_databases_found') @@ -264,7 +266,15 @@ def test_run_restore_restores_each_database(): hooks=flexmock(), local_borg_version=flexmock(), restore_arguments=flexmock( - repository='repo', archive='archive', databases=flexmock(), schemas=None + repository='repo', + archive='archive', + databases=flexmock(), + schemas=None, + hostname=None, + port=None, + username=None, + password=None, + restore_path=None, ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), @@ -337,6 +347,7 @@ def test_run_restore_restores_database_configured_with_all_name(): archive_name=object, hook_name='postgresql_databases', database={'name': 'foo', 'schemas': None}, + connection_params=object, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -350,6 +361,7 @@ def test_run_restore_restores_database_configured_with_all_name(): archive_name=object, hook_name='postgresql_databases', database={'name': 'bar', 'schemas': None}, + connection_params=object, ).once() flexmock(module).should_receive('ensure_databases_found') @@ -360,7 +372,15 @@ def test_run_restore_restores_database_configured_with_all_name(): hooks=flexmock(), local_borg_version=flexmock(), restore_arguments=flexmock( - repository='repo', archive='archive', databases=flexmock(), schemas=None + repository='repo', + archive='archive', + databases=flexmock(), + schemas=None, + hostname=None, + port=None, + username=None, + password=None, + restore_path=None, ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), @@ -411,6 +431,7 @@ def test_run_restore_skips_missing_database(): archive_name=object, hook_name='postgresql_databases', database={'name': 'foo', 'schemas': None}, + connection_params=object, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -424,6 +445,7 @@ def test_run_restore_skips_missing_database(): archive_name=object, hook_name='postgresql_databases', database={'name': 'bar', 'schemas': None}, + connection_params=object, ).never() flexmock(module).should_receive('ensure_databases_found') @@ -434,7 +456,15 @@ def test_run_restore_skips_missing_database(): hooks=flexmock(), local_borg_version=flexmock(), restore_arguments=flexmock( - repository='repo', archive='archive', databases=flexmock(), schemas=None + repository='repo', + archive='archive', + databases=flexmock(), + schemas=None, + hostname=None, + port=None, + username=None, + password=None, + restore_path=None, ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), @@ -479,6 +509,7 @@ def test_run_restore_restores_databases_from_different_hooks(): archive_name=object, hook_name='postgresql_databases', database={'name': 'foo', 'schemas': None}, + connection_params=object, ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, @@ -492,6 +523,7 @@ def test_run_restore_restores_databases_from_different_hooks(): archive_name=object, hook_name='mysql_databases', database={'name': 'bar', 'schemas': None}, + connection_params=object, ).once() flexmock(module).should_receive('ensure_databases_found') @@ -502,7 +534,15 @@ def test_run_restore_restores_databases_from_different_hooks(): hooks=flexmock(), local_borg_version=flexmock(), restore_arguments=flexmock( - repository='repo', archive='archive', databases=flexmock(), schemas=None + repository='repo', + archive='archive', + databases=flexmock(), + schemas=None, + hostname=None, + port=None, + username=None, + password=None, + restore_path=None, ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index f038a881..2c09fef6 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -171,7 +171,17 @@ def test_restore_database_dump_runs_mongorestore(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -185,7 +195,17 @@ def test_restore_database_dump_errors_on_multiple_database_config(): with pytest.raises(ValueError): module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock() + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=flexmock(), + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -215,7 +235,17 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -253,7 +283,17 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -271,7 +311,17 @@ def test_restore_database_dump_runs_mongorestore_with_options(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -299,7 +349,17 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -317,7 +377,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -329,7 +399,17 @@ def test_restore_database_dump_with_dry_run_skips_restore(): flexmock(module).should_receive('execute_command_with_processes').never() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock() + database_config, + 'test.yaml', + {}, + dry_run=True, + extract_process=flexmock(), + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -346,5 +426,15 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=None + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=None, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index da5da16d..9b65b121 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -392,7 +392,17 @@ def test_restore_database_dump_runs_mysql_to_restore(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -404,7 +414,17 @@ def test_restore_database_dump_errors_on_multiple_database_config(): with pytest.raises(ValueError): module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock() + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=flexmock(), + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -421,7 +441,17 @@ def test_restore_database_dump_runs_mysql_with_options(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -447,7 +477,17 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -464,7 +504,17 @@ def test_restore_database_dump_runs_mysql_with_username_and_password(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -474,5 +524,15 @@ def test_restore_database_dump_with_dry_run_skips_restore(): flexmock(module).should_receive('execute_command_with_processes').never() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock() + database_config, + 'test.yaml', + {}, + dry_run=True, + extract_process=flexmock(), + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index b3a55fa4..3baf4dc0 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -479,7 +479,17 @@ def test_restore_database_dump_runs_pg_restore(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -494,7 +504,17 @@ def test_restore_database_dump_errors_on_multiple_database_config(): with pytest.raises(ValueError): module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock() + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=flexmock(), + connection_params={ + 'restore_hostname': None, + 'restore_port': None, + 'restore_username': None, + 'restore_password': None, + }, ) @@ -545,7 +565,17 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -594,7 +624,17 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -644,7 +684,17 @@ def test_restore_database_dump_runs_pg_restore_with_options(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -672,7 +722,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -705,7 +765,17 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -759,7 +829,17 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -772,7 +852,17 @@ def test_restore_database_dump_with_dry_run_skips_restore(): flexmock(module).should_receive('execute_command_with_processes').never() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock() + database_config, + 'test.yaml', + {}, + dry_run=True, + extract_process=flexmock(), + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -813,7 +903,17 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=None + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=None, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) @@ -858,5 +958,15 @@ def test_restore_database_dump_with_schemas_restores_schemas(): ).once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=None + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=None, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, ) diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py index a660d818..2edd4d10 100644 --- a/tests/unit/hooks/test_sqlite.py +++ b/tests/unit/hooks/test_sqlite.py @@ -99,7 +99,12 @@ def test_restore_database_dump_restores_database(): flexmock(module.os).should_receive('remove').once() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': None}, ) @@ -111,7 +116,12 @@ def test_restore_database_dump_does_not_restore_database_if_dry_run(): flexmock(module.os).should_receive('remove').never() module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=True, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=True, + extract_process=extract_process, + connection_params={'restore_path': None}, ) @@ -121,5 +131,10 @@ def test_restore_database_dump_raises_error_if_database_config_is_invalid(): with pytest.raises(ValueError): module.restore_database_dump( - database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': None}, ) From ee2ebb79b8e89dbe161f0ba87d950b2dfac8c82f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 16 Jun 2023 10:57:01 -0700 Subject: [PATCH 274/344] Find sub-actions for an action without an isinstance() check. --- borgmatic/commands/completion/actions.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/borgmatic/commands/completion/actions.py b/borgmatic/commands/completion/actions.py index 903670da..690d0a88 100644 --- a/borgmatic/commands/completion/actions.py +++ b/borgmatic/commands/completion/actions.py @@ -1,6 +1,3 @@ -import argparse - - def upgrade_message(language: str, upgrade_command: str, completion_file: str): return f''' Your {language} completions script is from a different version of borgmatic than is @@ -25,11 +22,11 @@ def available_actions(subparsers, current_action=None): actions_to_subactions = { action: tuple( subaction_name - for subaction in subparser._actions - if isinstance(subaction, argparse._SubParsersAction) - for subaction_name in subaction.choices.keys() + for group_action in subparser._subparsers._group_actions + for subaction_name in group_action.choices.keys() ) for action, subparser in subparsers.choices.items() + if subparser._subparsers } current_subactions = actions_to_subactions.get(current_action) From 6c876085488b8ab9aef7cc9bab3da6534df790c1 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 17 Jun 2023 00:47:15 +0530 Subject: [PATCH 275/344] add tests for password logic --- tests/unit/hooks/test_postgresql.py | 108 ++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 3baf4dc0..30ed7e31 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -638,6 +638,114 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): ) +def test_restore_database_dump_with_cli_password_runs_pg_restore_with_password(): + database_config = [{'name': 'foo', 'username': 'postgres', 'schemas': None}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'pg_restore', + '--no-password', + '--if-exists', + '--exit-on-error', + '--clean', + '--dbname', + 'foo', + '--username', + 'postgres', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--username', + 'postgres', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), + extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + ).once() + + module.restore_database_dump( + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': 'trustsome1', + }, + ) + + +def test_restore_database_dump_with_no_passwords_runs_pg_restore_without_password(): + database_config = [{'name': 'foo', 'username': 'postgres', 'schemas': None}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'pg_restore', + '--no-password', + '--if-exists', + '--exit-on-error', + '--clean', + '--dbname', + 'foo', + '--username', + 'postgres', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--username', + 'postgres', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), + extra_environment={'PGSSLMODE': 'disable'}, + ).once() + + module.restore_database_dump( + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + def test_restore_database_dump_runs_pg_restore_with_options(): database_config = [ { From e53dd3da87bbd743061ad8e46dbf0a8a627f1918 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 17 Jun 2023 22:58:59 +0530 Subject: [PATCH 276/344] fix witten reported mysql error --- borgmatic/hooks/mysql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 22eef61e..aee13d80 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -216,10 +216,10 @@ def restore_database_dump( restore_command = ( ('mysql', '--batch') + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ()) - + (('--host', database['hostname']) if hostname else ()) - + (('--port', str(database['port'])) if port else ()) + + (('--host', hostname) if hostname else ()) + + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) - + (('--user', database['username']) if username else ()) + + (('--user', username) if username else ()) ) extra_environment = {'MYSQL_PWD': password} if password else None From 9016dcc41816fa5acd74f33c931aca330660abc1 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sun, 18 Jun 2023 05:47:35 +0530 Subject: [PATCH 277/344] all e2e tests --- borgmatic/config/schema.yaml | 1 + tests/end-to-end/docker-compose.yaml | 25 ++++ tests/end-to-end/test_database.py | 187 +++++++++++++++++++++++++++ 3 files changed, 213 insertions(+) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 7d079ff0..fb8d7b15 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -818,6 +818,7 @@ properties: These statements will fail unless the initial connection to the database is made by a superuser. + example: true format: type: string enum: ['plain', 'custom', 'directory', 'tar'] diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 0bbec8cc..0aa19c56 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -5,16 +5,38 @@ services: environment: POSTGRES_PASSWORD: test POSTGRES_DB: test + postgresql2: + image: docker.io/postgres:13.1-alpine + environment: + POSTGRES_PASSWORD: test2 + POSTGRES_DB: test + POSTGRES_USER: postgres2 + ports: + - "5433:5432" mysql: image: docker.io/mariadb:10.5 environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test + mysql2: + image: docker.io/mariadb:10.5 + environment: + MYSQL_ROOT_PASSWORD: test2 + MYSQL_DATABASE: test + ports: + - "3307:3306" mongodb: image: docker.io/mongo:5.0.5 environment: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: test + mongodb2: + image: docker.io/mongo:5.0.5 + environment: + MONGO_INITDB_ROOT_USERNAME: root2 + MONGO_INITDB_ROOT_PASSWORD: test2 + ports: + - "27018:27017" tests: image: docker.io/alpine:3.13 environment: @@ -30,5 +52,8 @@ services: command: --end-to-end-only depends_on: - postgresql + - postgresql2 - mysql + - mysql2 - mongodb + - mongodb2 diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 5c4e22cc..b180acb5 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -81,6 +81,107 @@ hooks: with open(config_path, 'w') as config_file: config_file.write(config) +def write_custom_restore_configuration( + source_directory, + config_path, + repository_path, + borgmatic_source_directory, + postgresql_dump_format='custom', + mongodb_dump_format='archive', +): + ''' + Write out borgmatic configuration into a file at the config path. Set the options so as to work + for testing with custom restore options. This includes a custom restore_hostname, restore_port, + restore_username, restore_password and restore_path. + ''' + config = f''' +location: + source_directories: + - {source_directory} + repositories: + - {repository_path} + borgmatic_source_directory: {borgmatic_source_directory} + +storage: + encryption_passphrase: "test" + +hooks: + postgresql_databases: + - name: test + hostname: postgresql + username: postgres + password: test + format: {postgresql_dump_format} + restore_hostname: postgresql2 + restore_port: 5432 + restore_username: postgres2 + restore_password: test2 + mysql_databases: + - name: test + hostname: mysql + username: root + password: test + restore_hostname: mysql2 + restore_port: 3306 + restore_username: root + restore_password: test2 + mongodb_databases: + - name: test + hostname: mongodb + username: root + password: test + authentication_database: admin + format: {mongodb_dump_format} + restore_hostname: mongodb2 + restore_port: 27017 + restore_username: root2 + restore_password: test2 + sqlite_databases: + - name: sqlite_test + path: /tmp/sqlite_test.db + restore_path: /tmp/sqlite_test2.db +''' + + with open(config_path, 'w') as config_file: + config_file.write(config) + + +def write_custom_restore_configuration_for_cli_arguments( + source_directory, + config_path, + repository_path, + borgmatic_source_directory, + postgresql_dump_format='custom', +): + ''' + Write out borgmatic configuration into a file at the config path. Set the options so as to work + for testing with custom restore options, but this time using CLI arguments. This includes a + custom restore_hostname, restore_port, restore_username and restore_password as we only test + these options for PostgreSQL. + ''' + config = f''' +location: + source_directories: + - {source_directory} + repositories: + - {repository_path} + borgmatic_source_directory: {borgmatic_source_directory} + +storage: + encryption_passphrase: "test" + +hooks: + postgresql_databases: + - name: test + hostname: postgresql + username: postgres + password: test + format: {postgresql_dump_format} +''' + + with open(config_path, 'w') as config_file: + config_file.write(config) + def test_database_dump_and_restore(): # Create a Borg repository. @@ -125,6 +226,92 @@ def test_database_dump_and_restore(): shutil.rmtree(temporary_directory) +def test_database_dump_and_restore_with_restore_cli_arguments(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic') + + # Write out a special file to ensure that it gets properly excluded and Borg doesn't hang on it. + os.mkfifo(os.path.join(temporary_directory, 'special_file')) + + original_working_directory = os.getcwd() + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + write_custom_restore_configuration_for_cli_arguments( + temporary_directory, config_path, repository_path, borgmatic_source_directory + ) + + subprocess.check_call( + ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] + ) + + # Run borgmatic to generate a backup archive including a database dump. + subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2']) + + # Get the created archive name. + output = subprocess.check_output( + ['borgmatic', '--config', config_path, 'list', '--json'] + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + archive_name = parsed_output[0]['archives'][0]['archive'] + + # Restore the database from the archive. + subprocess.check_call( + ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name, '--hostname', 'postgresql2', '--port', '5432', '--username', 'postgres2', '--password', 'test2'] + ) + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) + + +def test_database_dump_and_restore_to_different_hostname_port_username_password(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic') + + # Write out a special file to ensure that it gets properly excluded and Borg doesn't hang on it. + os.mkfifo(os.path.join(temporary_directory, 'special_file')) + + original_working_directory = os.getcwd() + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + write_custom_restore_configuration( + temporary_directory, config_path, repository_path, borgmatic_source_directory + ) + + subprocess.check_call( + ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] + ) + + # Run borgmatic to generate a backup archive including a database dump. + subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2']) + + # Get the created archive name. + output = subprocess.check_output( + ['borgmatic', '--config', config_path, 'list', '--json'] + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + archive_name = parsed_output[0]['archives'][0]['archive'] + + # Restore the database from the archive. + subprocess.check_call( + ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name] + ) + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) + + def test_database_dump_and_restore_with_directory_format(): # Create a Borg repository. temporary_directory = tempfile.mkdtemp() From 384182172a29c89c393786daeca1c9d71136ee1e Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sun, 18 Jun 2023 06:29:11 +0530 Subject: [PATCH 278/344] add unit tests for cases when cli/config restore args are used --- tests/end-to-end/test_database.py | 20 ++++- tests/unit/hooks/test_postgresql.py | 122 ++++++++++++++++++++++++++-- 2 files changed, 134 insertions(+), 8 deletions(-) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index b180acb5..c424a7e2 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -81,6 +81,7 @@ hooks: with open(config_path, 'w') as config_file: config_file.write(config) + def write_custom_restore_configuration( source_directory, config_path, @@ -262,7 +263,24 @@ def test_database_dump_and_restore_with_restore_cli_arguments(): # Restore the database from the archive. subprocess.check_call( - ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name, '--hostname', 'postgresql2', '--port', '5432', '--username', 'postgres2', '--password', 'test2'] + [ + 'borgmatic', + '-v', + '2', + '--config', + config_path, + 'restore', + '--archive', + archive_name, + '--hostname', + 'postgresql2', + '--port', + '5432', + '--username', + 'postgres2', + '--password', + 'test2', + ] ) finally: os.chdir(original_working_directory) diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 30ed7e31..60bbe57e 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -638,10 +638,32 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): ) -def test_restore_database_dump_with_cli_password_runs_pg_restore_with_password(): - database_config = [{'name': 'foo', 'username': 'postgres', 'schemas': None}] +def test_make_extra_environment_with_cli_password_sets_correct_password(): + database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'} + + extra = module.make_extra_environment( + database, restore_connection_params={'password': 'clipassword'} + ) + + assert extra['PGPASSWORD'] == 'clipassword' + + +def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): + database_config = [ + { + 'name': 'foo', + 'hostname': 'database.example.org', + 'port': 5433, + 'username': 'postgres', + 'password': 'trustsome1', + 'schemas': None, + } + ] extract_process = flexmock(stdout=flexmock()) + flexmock(module).should_receive('make_extra_environment').and_return( + {'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'} + ) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -653,13 +675,17 @@ def test_restore_database_dump_with_cli_password_runs_pg_restore_with_password() '--clean', '--dbname', 'foo', + '--host', + 'clihost', + '--port', + 'cliport', '--username', - 'postgres', + 'cliusername', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -667,14 +693,96 @@ def test_restore_database_dump_with_cli_password_runs_pg_restore_with_password() '--no-password', '--no-psqlrc', '--quiet', + '--host', + 'clihost', + '--port', + 'cliport', '--username', - 'postgres', + 'cliusername', '--dbname', 'foo', '--command', 'ANALYZE', ), - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, + ).once() + + module.restore_database_dump( + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': 'clihost', + 'port': 'cliport', + 'username': 'cliusername', + 'password': 'clipassword', + }, + ) + + +def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore(): + database_config = [ + { + 'name': 'foo', + 'hostname': 'database.example.org', + 'port': 5433, + 'username': 'postgres', + 'password': 'trustsome1', + 'schemas': None, + 'restore_hostname': 'restorehost', + 'restore_port': 'restoreport', + 'restore_username': 'restoreusername', + 'restore_password': 'restorepassword', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_extra_environment').and_return( + {'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'} + ) + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'pg_restore', + '--no-password', + '--if-exists', + '--exit-on-error', + '--clean', + '--dbname', + 'foo', + '--host', + 'restorehost', + '--port', + 'restoreport', + '--username', + 'restoreusername', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, + ).once() + flexmock(module).should_receive('execute_command').with_args( + ( + 'psql', + '--no-password', + '--no-psqlrc', + '--quiet', + '--host', + 'restorehost', + '--port', + 'restoreport', + '--username', + 'restoreusername', + '--dbname', + 'foo', + '--command', + 'ANALYZE', + ), + extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, ).once() module.restore_database_dump( @@ -687,7 +795,7 @@ def test_restore_database_dump_with_cli_password_runs_pg_restore_with_password() 'hostname': None, 'port': None, 'username': None, - 'password': 'trustsome1', + 'password': None, }, ) From e2d82e9bba1fb053eee717c478df4aaea8629cc9 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Mon, 19 Jun 2023 01:10:01 +0530 Subject: [PATCH 279/344] actually test port restores --- tests/end-to-end/docker-compose.yaml | 3 ++ tests/end-to-end/test_database.py | 20 ++++------- tests/unit/hooks/test_postgresql.py | 54 ---------------------------- 3 files changed, 10 insertions(+), 67 deletions(-) diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 0aa19c56..a769c16f 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -13,6 +13,7 @@ services: POSTGRES_USER: postgres2 ports: - "5433:5432" + command: -p 5433 mysql: image: docker.io/mariadb:10.5 environment: @@ -25,6 +26,7 @@ services: MYSQL_DATABASE: test ports: - "3307:3306" + command: --port=3307 mongodb: image: docker.io/mongo:5.0.5 environment: @@ -37,6 +39,7 @@ services: MONGO_INITDB_ROOT_PASSWORD: test2 ports: - "27018:27017" + command: --port=27018 tests: image: docker.io/alpine:3.13 environment: diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index c424a7e2..fd565b21 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -114,7 +114,7 @@ hooks: password: test format: {postgresql_dump_format} restore_hostname: postgresql2 - restore_port: 5432 + restore_port: 5433 restore_username: postgres2 restore_password: test2 mysql_databases: @@ -123,7 +123,7 @@ hooks: username: root password: test restore_hostname: mysql2 - restore_port: 3306 + restore_port: 3307 restore_username: root restore_password: test2 mongodb_databases: @@ -134,7 +134,7 @@ hooks: authentication_database: admin format: {mongodb_dump_format} restore_hostname: mongodb2 - restore_port: 27017 + restore_port: 27018 restore_username: root2 restore_password: test2 sqlite_databases: @@ -147,7 +147,7 @@ hooks: config_file.write(config) -def write_custom_restore_configuration_for_cli_arguments( +def write_simple_custom_restore_configuration( source_directory, config_path, repository_path, @@ -233,14 +233,11 @@ def test_database_dump_and_restore_with_restore_cli_arguments(): repository_path = os.path.join(temporary_directory, 'test.borg') borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic') - # Write out a special file to ensure that it gets properly excluded and Borg doesn't hang on it. - os.mkfifo(os.path.join(temporary_directory, 'special_file')) - original_working_directory = os.getcwd() try: config_path = os.path.join(temporary_directory, 'test.yaml') - write_custom_restore_configuration_for_cli_arguments( + write_simple_custom_restore_configuration( temporary_directory, config_path, repository_path, borgmatic_source_directory ) @@ -275,7 +272,7 @@ def test_database_dump_and_restore_with_restore_cli_arguments(): '--hostname', 'postgresql2', '--port', - '5432', + '5433', '--username', 'postgres2', '--password', @@ -287,15 +284,12 @@ def test_database_dump_and_restore_with_restore_cli_arguments(): shutil.rmtree(temporary_directory) -def test_database_dump_and_restore_to_different_hostname_port_username_password(): +def test_database_dump_and_restore_with_restore_configuration_options(): # Create a Borg repository. temporary_directory = tempfile.mkdtemp() repository_path = os.path.join(temporary_directory, 'test.borg') borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic') - # Write out a special file to ensure that it gets properly excluded and Borg doesn't hang on it. - os.mkfifo(os.path.join(temporary_directory, 'special_file')) - original_working_directory = os.getcwd() try: diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 60bbe57e..5120ad02 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -800,60 +800,6 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ ) -def test_restore_database_dump_with_no_passwords_runs_pg_restore_without_password(): - database_config = [{'name': 'foo', 'username': 'postgres', 'schemas': None}] - extract_process = flexmock(stdout=flexmock()) - - flexmock(module).should_receive('make_dump_path') - flexmock(module.dump).should_receive('make_database_dump_filename') - flexmock(module).should_receive('execute_command_with_processes').with_args( - ( - 'pg_restore', - '--no-password', - '--if-exists', - '--exit-on-error', - '--clean', - '--dbname', - 'foo', - '--username', - 'postgres', - ), - processes=[extract_process], - output_log_level=logging.DEBUG, - input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, - ).once() - flexmock(module).should_receive('execute_command').with_args( - ( - 'psql', - '--no-password', - '--no-psqlrc', - '--quiet', - '--username', - 'postgres', - '--dbname', - 'foo', - '--command', - 'ANALYZE', - ), - extra_environment={'PGSSLMODE': 'disable'}, - ).once() - - module.restore_database_dump( - database_config, - 'test.yaml', - {}, - dry_run=False, - extract_process=extract_process, - connection_params={ - 'hostname': None, - 'port': None, - 'username': None, - 'password': None, - }, - ) - - def test_restore_database_dump_runs_pg_restore_with_options(): database_config = [ { From 1a21eb03cdc4e506575a6579b36ff758b7aa069d Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 20 Jun 2023 00:52:01 +0530 Subject: [PATCH 280/344] add tests for all databases --- tests/end-to-end/docker-compose.yaml | 6 -- tests/unit/hooks/test_mongodb.py | 108 +++++++++++++++++++++++++++ tests/unit/hooks/test_mysql.py | 88 ++++++++++++++++++++++ tests/unit/hooks/test_sqlite.py | 65 +++++++++++++++- 4 files changed, 260 insertions(+), 7 deletions(-) diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index a769c16f..bbeb29f2 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -11,8 +11,6 @@ services: POSTGRES_PASSWORD: test2 POSTGRES_DB: test POSTGRES_USER: postgres2 - ports: - - "5433:5432" command: -p 5433 mysql: image: docker.io/mariadb:10.5 @@ -24,8 +22,6 @@ services: environment: MYSQL_ROOT_PASSWORD: test2 MYSQL_DATABASE: test - ports: - - "3307:3306" command: --port=3307 mongodb: image: docker.io/mongo:5.0.5 @@ -37,8 +33,6 @@ services: environment: MONGO_INITDB_ROOT_USERNAME: root2 MONGO_INITDB_ROOT_PASSWORD: test2 - ports: - - "27018:27017" command: --port=27018 tests: image: docker.io/alpine:3.13 diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 2c09fef6..14f7cafb 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -297,6 +297,114 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): ) +def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): + database_config = [ + { + 'name': 'foo', + 'username': 'mongo', + 'password': 'trustsome1', + 'authentication_database': 'admin', + 'schemas': None, + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + [ + 'mongorestore', + '--archive', + '--drop', + '--db', + 'foo', + '--host', + 'clihost', + '--port', + 'cliport', + '--username', + 'cliusername', + '--password', + 'clipassword', + '--authenticationDatabase', + 'admin', + ], + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + module.restore_database_dump( + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': 'clihost', + 'port': 'cliport', + 'username': 'cliusername', + 'password': 'clipassword', + }, + ) + + +def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore(): + database_config = [ + { + 'name': 'foo', + 'username': 'mongo', + 'password': 'trustsome1', + 'authentication_database': 'admin', + 'schemas': None, + 'restore_hostname': 'restorehost', + 'restore_port': 'restoreport', + 'restore_username': 'restoreuser', + 'restore_password': 'restorepass', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_database_dump_filename') + flexmock(module).should_receive('execute_command_with_processes').with_args( + [ + 'mongorestore', + '--archive', + '--drop', + '--db', + 'foo', + '--host', + 'restorehost', + '--port', + 'restoreport', + '--username', + 'restoreuser', + '--password', + 'restorepass', + '--authenticationDatabase', + 'admin', + ], + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + module.restore_database_dump( + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + def test_restore_database_dump_runs_mongorestore_with_options(): database_config = [{'name': 'foo', 'restore_options': '--harder', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 9b65b121..e45fa56e 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -518,6 +518,94 @@ def test_restore_database_dump_runs_mysql_with_username_and_password(): ) +def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): + database_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mysql', + '--batch', + '--host', + 'clihost', + '--port', + 'cliport', + '--protocol', + 'tcp', + '--user', + 'cliusername', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'MYSQL_PWD': 'clipassword'}, + ).once() + + module.restore_database_dump( + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': 'clihost', + 'port': 'cliport', + 'username': 'cliusername', + 'password': 'clipassword', + }, + ) + + +def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore(): + database_config = [ + { + 'name': 'foo', + 'username': 'root', + 'password': 'trustsome1', + 'hostname': 'dbhost', + 'port': 'dbport', + 'restore_username': 'restoreuser', + 'restore_password': 'restorepass', + 'restore_hostname': 'restorehost', + 'restore_port': 'restoreport', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mysql', + '--batch', + '--host', + 'restorehost', + '--port', + 'restoreport', + '--protocol', + 'tcp', + '--user', + 'restoreuser', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + extra_environment={'MYSQL_PWD': 'restorepass'}, + ).once() + + module.restore_database_dump( + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + ) + + def test_restore_database_dump_with_dry_run_skips_restore(): database_config = [{'name': 'foo'}] diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py index 2edd4d10..30700e6e 100644 --- a/tests/unit/hooks/test_sqlite.py +++ b/tests/unit/hooks/test_sqlite.py @@ -1,3 +1,4 @@ +import logging import pytest from flexmock import flexmock @@ -94,7 +95,69 @@ def test_restore_database_dump_restores_database(): database_config = [{'path': '/path/to/database', 'name': 'database'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('execute_command_with_processes').once() + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'sqlite3', + '/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_database_dump( + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': None}, + ) + + +def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): + database_config = [{'path': '/path/to/database', 'name': 'database'}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'sqlite3', + 'cli/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_database_dump( + database_config, + 'test.yaml', + {}, + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': 'cli/path/to/database'}, + ) + + +def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore(): + database_config = [ + {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'sqlite3', + 'config/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() flexmock(module.os).should_receive('remove').once() From 7b8be800a4a9788e942938775e5071195a614686 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 19 Jun 2023 16:18:47 -0700 Subject: [PATCH 281/344] Refactor arguments parsing to fix bootstrap action CLI issues (#712). --- NEWS | 4 +- borgmatic/borg/borg.py | 5 +- borgmatic/commands/arguments.py | 377 +++++++------- borgmatic/commands/borgmatic.py | 2 +- borgmatic/commands/completion/actions.py | 21 +- tests/integration/commands/test_arguments.py | 72 ++- tests/unit/commands/test_arguments.py | 498 +++++++++++++------ tests/unit/commands/test_borgmatic.py | 2 +- 8 files changed, 617 insertions(+), 364 deletions(-) diff --git a/NEWS b/NEWS index 6f230e43..50391a87 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,7 @@ 1.7.15.dev0 * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors. - * #697: Extract borgmatic configuration from backup via "bootstrap" action—even when borgmatic - has no configuration yet! + * #697, #712: Extract borgmatic configuration from backup via "bootstrap" action—even when + borgmatic has no configuration yet! * #669: Add sample systemd user service for running borgmatic as a non-root user. 1.7.14 diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index 82fecc1a..f815dfd0 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -38,10 +38,7 @@ def run_arbitrary_borg( borg_command = tuple(options[:command_options_start_index]) command_options = tuple(options[command_options_start_index:]) - if ( - borg_command - and borg_command[0] in borgmatic.commands.arguments.SUBPARSER_ALIASES.keys() - ): + if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES.keys(): logger.warning( f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}" ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 30ac3f9c..2f978a3f 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,18 +1,17 @@ -import argparse import collections import itertools +import sys from argparse import Action, ArgumentParser from borgmatic.config import collect -SUBPARSER_ALIASES = { +ACTION_ALIASES = { 'rcreate': ['init', '-I'], 'prune': ['-p'], 'compact': [], 'create': ['-C'], 'check': ['-k'], 'config': [], - 'config_bootstrap': [], 'extract': ['-x'], 'export-tar': [], 'mount': ['-m'], @@ -28,124 +27,192 @@ SUBPARSER_ALIASES = { } -def get_unparsable_arguments(remaining_subparser_arguments): +def get_subaction_parsers(action_parser): ''' - Determine the remaining arguments that no subparsers have consumed. + Given an argparse.ArgumentParser instance, lookup the subactions in it and return a dict from + subaction name to subaction parser. ''' - if remaining_subparser_arguments: - remaining_arguments = [ - argument - for argument in dict.fromkeys( - itertools.chain.from_iterable(remaining_subparser_arguments) - ).keys() - if all( - argument in subparser_arguments - for subparser_arguments in remaining_subparser_arguments - ) - ] - else: - remaining_arguments = [] + if not action_parser._subparsers: + return {} - return remaining_arguments - - -def parse_subparser_arguments(unparsed_arguments, subparsers): - ''' - Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser - instance, give each requested action's subparser a shot at parsing all arguments. This allows - common arguments like "--repository" to be shared across multiple subparsers. - - Return the result as a tuple of (a dict mapping from subparser name to a parsed namespace of - arguments, a list of remaining arguments not claimed by any subparser). - ''' - arguments = collections.OrderedDict() - remaining_arguments = list(unparsed_arguments) - alias_to_subparser_name = { - alias: subparser_name - for subparser_name, aliases in SUBPARSER_ALIASES.items() - for alias in aliases - } - subcommand_parsers_mapping = { - 'config': ['bootstrap'], + return { + subaction_name: subaction_parser + for group_action in action_parser._subparsers._group_actions + for subaction_name, subaction_parser in group_action.choices.items() } - # If the "borg" action is used, skip all other subparsers. This avoids confusion like - # "borg list" triggering borgmatic's own list action. - if 'borg' in unparsed_arguments: - subparsers = {'borg': subparsers['borg']} - for argument in remaining_arguments: - canonical_name = alias_to_subparser_name.get(argument, argument) - subparser = subparsers.get(canonical_name) - - if not subparser: - continue - - # If a parsed value happens to be the same as the name of a subparser, remove it from the - # remaining arguments. This prevents, for instance, "check --only extract" from triggering - # the "extract" subparser. - parsed, unused_remaining = subparser.parse_known_args( - [argument for argument in unparsed_arguments if argument != canonical_name] +def get_subactions_for_actions(action_parsers): + ''' + Given a dict from action name to an argparse.ArgumentParser instance, make a map from action + name to the names of contained sub-actions. + ''' + return { + action: tuple( + subaction_name + for group_action in action_parser._subparsers._group_actions + for subaction_name in group_action.choices.keys() ) + for action, action_parser in action_parsers.items() + if action_parser._subparsers + } + + +def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments): + ''' + Given a sequence of string arguments and a dict from action name to parsed argparse.Namespace + arguments, return the string arguments with any values omitted that happen to be the same as + the name of a borgmatic action. + + This prevents, for instance, "check --only extract" from triggering the "extract" action. + ''' + remaining_arguments = list(unparsed_arguments) + + for action_name, parsed in parsed_arguments.items(): for value in vars(parsed).values(): if isinstance(value, str): - if value in subparsers: + if value in ACTION_ALIASES.keys(): remaining_arguments.remove(value) elif isinstance(value, list): for item in value: - if item in subparsers: + if item in ACTION_ALIASES.keys(): remaining_arguments.remove(item) - arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed + return tuple(remaining_arguments) - for argument in arguments: - if not arguments[argument]: - if not any( - subcommand in arguments for subcommand in subcommand_parsers_mapping[argument] - ): - raise ValueError( - f'Missing subcommand for {argument}. Expected one of {subcommand_parsers_mapping[argument]}' - ) - # If no actions are explicitly requested, assume defaults. - if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments: - for subparser_name in ('create', 'prune', 'compact', 'check'): - subparser = subparsers[subparser_name] - parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments) - arguments[subparser_name] = parsed +def parse_and_record_action_arguments( + unparsed_arguments, parsed_arguments, action_parser, action_name, canonical_name=None +): + ''' + Given unparsed arguments as a sequence of strings, parsed arguments as a dict from action name + to parsed argparse.Namespace, a parser to parse with, an action name, and an optional canonical + action name (in case this the action name is an alias), parse the arguments and return a list of + any remaining string arguments that were not parsed. Also record the parsed argparse.Namespace + by setting it into the given parsed arguments. Return None if no parsing occurs because the + given action doesn't apply to the given unparsed arguments. + ''' + filtered_arguments = omit_values_colliding_with_action_names( + unparsed_arguments, parsed_arguments + ) - remaining_arguments = list(unparsed_arguments) + if action_name not in filtered_arguments: + return tuple(unparsed_arguments) - # Now ask each subparser, one by one, to greedily consume arguments, from last to first. This - # allows subparsers to consume arguments before their parent subparsers do. - remaining_subparser_arguments = [] + parsed, remaining = action_parser.parse_known_args(filtered_arguments) + parsed_arguments[canonical_name or action_name] = parsed - for subparser_name, subparser in reversed(subparsers.items()): - if subparser_name not in arguments.keys(): + # Special case: If this is a "borg" action, greedily consume all arguments after (+1) the "borg" + # argument. + if action_name == 'borg': + borg_options_index = remaining.index('borg') + 1 + parsed_arguments['borg'].options = remaining[borg_options_index:] + remaining = remaining[:borg_options_index] + + return tuple(argument for argument in remaining if argument != action_name) + + +def get_unparsable_arguments(remaining_action_arguments): + ''' + Given a sequence of argument tuples (one tuple per action parser that parsed arguments), + determine the remaining arguments that no action parsers have consumed. + ''' + if not remaining_action_arguments: + return () + + return tuple( + argument + for argument in dict.fromkeys( + itertools.chain.from_iterable(remaining_action_arguments) + ).keys() + if all(argument in action_arguments for action_arguments in remaining_action_arguments) + ) + + +def parse_arguments_for_actions(unparsed_arguments, action_parsers): + ''' + Given a sequence of arguments and a dict from action name to argparse.ArgumentParser + instance, give each requested action's parser a shot at parsing all arguments. This allows + common arguments like "--repository" to be shared across multiple action parsers. + + Return the result as a tuple of: (a dict mapping from action name to an argparse.Namespace of + parsed arguments, a list of strings of remaining arguments not claimed by any action parser). + ''' + arguments = collections.OrderedDict() + help_requested = bool('--help' in unparsed_arguments or '-h' in unparsed_arguments) + remaining_action_arguments = [] + alias_to_action_name = { + alias: action_name for action_name, aliases in ACTION_ALIASES.items() for alias in aliases + } + + # If the "borg" action is used, skip all other action parsers. This avoids confusion like + # "borg list" triggering borgmatic's own list action. + if 'borg' in unparsed_arguments: + action_parsers = {'borg': action_parsers['borg']} + + # Ask each action parser, one by one, to parse arguments. + for argument in unparsed_arguments: + action_name = argument + canonical_name = alias_to_action_name.get(action_name, action_name) + action_parser = action_parsers.get(action_name) + + if not action_parser: continue - subparser = subparsers[subparser_name] - unused_parsed, remaining = subparser.parse_known_args( - [argument for argument in unparsed_arguments if argument != subparser_name] - ) - remaining_subparser_arguments.append(remaining) + subaction_parsers = get_subaction_parsers(action_parser) - if remaining_subparser_arguments: - remaining_arguments = get_unparsable_arguments(remaining_subparser_arguments) + # Parse with subaction parsers, if any. + if subaction_parsers: + subactions_parsed = False - # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the - # "borg" action. - if 'borg' in arguments: - borg_options_index = remaining_arguments.index('borg') + 1 - arguments['borg'].options = remaining_arguments[borg_options_index:] - remaining_arguments = remaining_arguments[:borg_options_index] + for subaction_name, subaction_parser in subaction_parsers.items(): + remaining_action_arguments.append( + parse_and_record_action_arguments( + unparsed_arguments, + arguments, + subaction_parser, + subaction_name, + ) + ) - # Remove the subparser names themselves. - for subparser_name, subparser in subparsers.items(): - if subparser_name in remaining_arguments: - remaining_arguments.remove(subparser_name) + if subaction_name in arguments: + subactions_parsed = True - return (arguments, remaining_arguments) + if not subactions_parsed: + if help_requested: + action_parser.print_help() + sys.exit(0) + else: + raise ValueError( + f"Missing sub-action after {action_name} action. Expected one of: {', '.join(get_subactions_for_actions(action_parsers)[action_name])}" + ) + # Otherwise, parse with the main action parser. + else: + remaining_action_arguments.append( + parse_and_record_action_arguments( + unparsed_arguments, arguments, action_parser, action_name, canonical_name + ) + ) + + # If no actions were explicitly requested, assume defaults. + if not arguments and not help_requested: + for default_action_name in ('create', 'prune', 'compact', 'check'): + default_action_parser = action_parsers[default_action_name] + remaining_action_arguments.append( + parse_and_record_action_arguments( + tuple(unparsed_arguments) + (default_action_name,), + arguments, + default_action_parser, + default_action_name, + ) + ) + + return ( + arguments, + get_unparsable_arguments(tuple(remaining_action_arguments)) + if arguments + else unparsed_arguments, + ) class Extend_action(Action): @@ -164,7 +231,7 @@ class Extend_action(Action): def make_parsers(): ''' - Build a top-level parser and its subparsers and return them as a tuple. + Build a top-level parser and its action parsers and return them as a tuple. ''' config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) @@ -283,14 +350,14 @@ def make_parsers(): parents=[global_parser], ) - subparsers = top_level_parser.add_subparsers( + action_parsers = top_level_parser.add_subparsers( title='actions', metavar='', help='Specify zero or more actions. Defaults to create, prune, compact, and check. Use --help with action for details:', ) - rcreate_parser = subparsers.add_parser( + rcreate_parser = action_parsers.add_parser( 'rcreate', - aliases=SUBPARSER_ALIASES['rcreate'], + aliases=ACTION_ALIASES['rcreate'], help='Create a new, empty Borg repository', description='Create a new, empty Borg repository', add_help=False, @@ -336,9 +403,9 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) - transfer_parser = subparsers.add_parser( + transfer_parser = action_parsers.add_parser( 'transfer', - aliases=SUBPARSER_ALIASES['transfer'], + aliases=ACTION_ALIASES['transfer'], help='Transfer archives from one repository to another, optionally upgrading the transferred data [Borg 2.0+ only]', description='Transfer archives from one repository to another, optionally upgrading the transferred data [Borg 2.0+ only]', add_help=False, @@ -409,9 +476,9 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) - prune_parser = subparsers.add_parser( + prune_parser = action_parsers.add_parser( 'prune', - aliases=SUBPARSER_ALIASES['prune'], + aliases=ACTION_ALIASES['prune'], help='Prune archives according to the retention policy (with Borg 1.2+, run compact afterwards to actually free space)', description='Prune archives according to the retention policy (with Borg 1.2+, run compact afterwards to actually free space)', add_help=False, @@ -453,9 +520,9 @@ def make_parsers(): ) prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - compact_parser = subparsers.add_parser( + compact_parser = action_parsers.add_parser( 'compact', - aliases=SUBPARSER_ALIASES['compact'], + aliases=ACTION_ALIASES['compact'], help='Compact segments to free space [Borg 1.2+, borgmatic 1.5.23+ only]', description='Compact segments to free space [Borg 1.2+, borgmatic 1.5.23+ only]', add_help=False, @@ -489,9 +556,9 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) - create_parser = subparsers.add_parser( + create_parser = action_parsers.add_parser( 'create', - aliases=SUBPARSER_ALIASES['create'], + aliases=ACTION_ALIASES['create'], help='Create an archive (actually perform a backup)', description='Create an archive (actually perform a backup)', add_help=False, @@ -523,9 +590,9 @@ def make_parsers(): ) create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - check_parser = subparsers.add_parser( + check_parser = action_parsers.add_parser( 'check', - aliases=SUBPARSER_ALIASES['check'], + aliases=ACTION_ALIASES['check'], help='Check archives for consistency', description='Check archives for consistency', add_help=False, @@ -565,9 +632,9 @@ def make_parsers(): ) check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - extract_parser = subparsers.add_parser( + extract_parser = action_parsers.add_parser( 'extract', - aliases=SUBPARSER_ALIASES['extract'], + aliases=ACTION_ALIASES['extract'], help='Extract files from a named archive to the current directory', description='Extract a named archive to the current directory', add_help=False, @@ -611,9 +678,9 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) - config_parser = subparsers.add_parser( + config_parser = action_parsers.add_parser( 'config', - aliases=SUBPARSER_ALIASES['config'], + aliases=ACTION_ALIASES['config'], help='Perform configuration file related operations', description='Perform configuration file related operations', add_help=False, @@ -622,15 +689,14 @@ def make_parsers(): config_group = config_parser.add_argument_group('config arguments') config_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - config_subparsers = config_parser.add_subparsers( - title='config subcommands', - description='Valid subcommands for config', + config_parsers = config_parser.add_subparsers( + title='config sub-actions', + description='Valid sub-actions for config', help='Additional help', ) - config_bootstrap_parser = config_subparsers.add_parser( + config_bootstrap_parser = config_parsers.add_parser( 'bootstrap', - aliases=SUBPARSER_ALIASES['config_bootstrap'], help='Extract the config files used to create a borgmatic repository', description='Extract config files that were used to create a borgmatic repository during the "create" action', add_help=False, @@ -676,9 +742,9 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) - export_tar_parser = subparsers.add_parser( + export_tar_parser = action_parsers.add_parser( 'export-tar', - aliases=SUBPARSER_ALIASES['export-tar'], + aliases=ACTION_ALIASES['export-tar'], help='Export an archive to a tar-formatted file or stream', description='Export an archive to a tar-formatted file or stream', add_help=False, @@ -722,9 +788,9 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) - mount_parser = subparsers.add_parser( + mount_parser = action_parsers.add_parser( 'mount', - aliases=SUBPARSER_ALIASES['mount'], + aliases=ACTION_ALIASES['mount'], help='Mount files from a named archive as a FUSE filesystem', description='Mount a named archive as a FUSE filesystem', add_help=False, @@ -787,9 +853,9 @@ def make_parsers(): mount_group.add_argument('--options', dest='options', help='Extra Borg mount options') mount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - umount_parser = subparsers.add_parser( + umount_parser = action_parsers.add_parser( 'umount', - aliases=SUBPARSER_ALIASES['umount'], + aliases=ACTION_ALIASES['umount'], help='Unmount a FUSE filesystem that was mounted with "borgmatic mount"', description='Unmount a mounted FUSE filesystem', add_help=False, @@ -804,9 +870,9 @@ def make_parsers(): ) umount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - restore_parser = subparsers.add_parser( + restore_parser = action_parsers.add_parser( 'restore', - aliases=SUBPARSER_ALIASES['restore'], + aliases=ACTION_ALIASES['restore'], help='Restore database dumps from a named archive', description='Restore database dumps from a named archive. (To extract files instead, use "borgmatic extract".)', add_help=False, @@ -837,9 +903,9 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) - rlist_parser = subparsers.add_parser( + rlist_parser = action_parsers.add_parser( 'rlist', - aliases=SUBPARSER_ALIASES['rlist'], + aliases=ACTION_ALIASES['rlist'], help='List repository', description='List the archives in a repository', add_help=False, @@ -897,9 +963,9 @@ def make_parsers(): ) rlist_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - list_parser = subparsers.add_parser( + list_parser = action_parsers.add_parser( 'list', - aliases=SUBPARSER_ALIASES['list'], + aliases=ACTION_ALIASES['list'], help='List archive', description='List the files in an archive or search for a file across archives', add_help=False, @@ -970,9 +1036,9 @@ def make_parsers(): ) list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - rinfo_parser = subparsers.add_parser( + rinfo_parser = action_parsers.add_parser( 'rinfo', - aliases=SUBPARSER_ALIASES['rinfo'], + aliases=ACTION_ALIASES['rinfo'], help='Show repository summary information such as disk space used', description='Show repository summary information such as disk space used', add_help=False, @@ -987,9 +1053,9 @@ def make_parsers(): ) rinfo_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - info_parser = subparsers.add_parser( + info_parser = action_parsers.add_parser( 'info', - aliases=SUBPARSER_ALIASES['info'], + aliases=ACTION_ALIASES['info'], help='Show archive summary information such as disk space used', description='Show archive summary information such as disk space used', add_help=False, @@ -1048,9 +1114,9 @@ def make_parsers(): ) info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - break_lock_parser = subparsers.add_parser( + break_lock_parser = action_parsers.add_parser( 'break-lock', - aliases=SUBPARSER_ALIASES['break-lock'], + aliases=ACTION_ALIASES['break-lock'], help='Break the repository and cache locks left behind by Borg aborting', description='Break Borg repository and cache locks left behind by Borg aborting', add_help=False, @@ -1064,9 +1130,9 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) - borg_parser = subparsers.add_parser( + borg_parser = action_parsers.add_parser( 'borg', - aliases=SUBPARSER_ALIASES['borg'], + aliases=ACTION_ALIASES['borg'], help='Run an arbitrary Borg command', description="Run an arbitrary Borg command based on borgmatic's configuration", add_help=False, @@ -1086,42 +1152,21 @@ def make_parsers(): ) borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - merged_subparsers = merge_subparsers(subparsers, config_subparsers) - - return top_level_parser, merged_subparsers - - -def merge_subparsers(*subparsers): - ''' - Merge multiple subparsers into a single subparser. - ''' - merged_subparsers = argparse._SubParsersAction( - None, None, metavar=None, dest='merged', parser_class=None - ) - - for subparser in subparsers: - for name, subparser in subparser.choices.items(): - merged_subparsers._name_parser_map[name] = subparser - - return merged_subparsers + return top_level_parser, action_parsers def parse_arguments(*unparsed_arguments): ''' Given command-line arguments with which this script was invoked, parse the arguments and return - them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance. + them as a dict mapping from action name (or "global") to an argparse.Namespace instance. ''' - top_level_parser, subparsers = make_parsers() + top_level_parser, action_parsers = make_parsers() - arguments, remaining_arguments = parse_subparser_arguments( - unparsed_arguments, subparsers.choices + arguments, remaining_arguments = parse_arguments_for_actions( + unparsed_arguments, action_parsers.choices ) - if ( - 'bootstrap' in arguments.keys() - and 'config' in arguments.keys() - and len(arguments.keys()) > 2 - ): + if 'bootstrap' in arguments.keys() and len(arguments.keys()) > 1: raise ValueError( 'The bootstrap action cannot be combined with other actions. Please run it separately.' ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c670291f..babfba0b 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -621,7 +621,7 @@ def collect_configuration_run_summary_logs(configs, arguments): ) yield logging.makeLogRecord( dict( - levelno=logging.INFO, + levelno=logging.ANSWER, levelname='INFO', msg='Bootstrap successful', ) diff --git a/borgmatic/commands/completion/actions.py b/borgmatic/commands/completion/actions.py index 690d0a88..1f0e2f36 100644 --- a/borgmatic/commands/completion/actions.py +++ b/borgmatic/commands/completion/actions.py @@ -1,3 +1,6 @@ +import borgmatic.commands.arguments + + def upgrade_message(language: str, upgrade_command: str, completion_file: str): return f''' Your {language} completions script is from a different version of borgmatic than is @@ -18,24 +21,16 @@ def available_actions(subparsers, current_action=None): "bootstrap" is a sub-action for "config", then "bootstrap" should be able to follow a current action of "config" but not "list". ''' - # Make a map from action name to the names of contained sub-actions. - actions_to_subactions = { - action: tuple( - subaction_name - for group_action in subparser._subparsers._group_actions - for subaction_name in group_action.choices.keys() - ) - for action, subparser in subparsers.choices.items() - if subparser._subparsers - } - - current_subactions = actions_to_subactions.get(current_action) + action_to_subactions = borgmatic.commands.arguments.get_subactions_for_actions( + subparsers.choices + ) + current_subactions = action_to_subactions.get(current_action) if current_subactions: return current_subactions all_subactions = set( - subaction for subactions in actions_to_subactions.values() for subaction in subactions + subaction for subactions in action_to_subactions.values() for subaction in subactions ) return tuple(action for action in subparsers.choices.keys() if action not in all_subactions) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 60fe884c..b2b03849 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -1,5 +1,3 @@ -import argparse - import pytest from flexmock import flexmock @@ -534,24 +532,62 @@ def test_parse_arguments_extract_with_check_only_extract_does_not_raise(): module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract') -def test_merging_two_subparser_collections_merges_their_choices(): - top_level_parser = argparse.ArgumentParser() +def test_parse_arguments_bootstrap_without_config_errors(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - subparsers = top_level_parser.add_subparsers() - subparser1 = subparsers.add_parser('subparser1') + with pytest.raises(SystemExit) as exit: + module.parse_arguments('bootstrap') - subparser2 = subparsers.add_parser('subparser2') - subsubparsers = subparser2.add_subparsers() - subsubparser1 = subsubparsers.add_parser('subsubparser1') + assert exit.value.code == 2 - merged_subparsers = argparse._SubParsersAction( - None, None, metavar=None, dest='merged', parser_class=None - ) - merged_subparsers = module.merge_subparsers(subparsers, subsubparsers) +def test_parse_arguments_config_with_no_subaction_errors(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - assert merged_subparsers.choices == { - 'subparser1': subparser1, - 'subparser2': subparser2, - 'subsubparser1': subsubparser1, - } + with pytest.raises(ValueError): + module.parse_arguments('config') + + +def test_parse_arguments_config_with_help_shows_config_help(capsys): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(SystemExit) as exit: + module.parse_arguments('config', '--help') + + assert exit.value.code == 0 + captured = capsys.readouterr() + assert 'global arguments:' not in captured.out + assert 'config arguments:' in captured.out + assert 'config sub-actions:' in captured.out + + +def test_parse_arguments_config_with_subaction_but_missing_flags_errors(capsys): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(SystemExit) as exit: + module.parse_arguments('config', 'bootstrap') + + assert exit.value.code == 2 + + +def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(capsys): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(SystemExit) as exit: + module.parse_arguments('config', 'bootstrap', '--help') + + assert exit.value.code == 0 + captured = capsys.readouterr() + assert 'config bootstrap arguments:' in captured.out + + +def test_parse_arguments_config_with_subaction_and_required_flags_does_not_raise(capsys): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg') + + +def test_parse_arguments_config_with_subaction_and_global_flags_does_not_raise(capsys): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + module.parse_arguments('--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg') diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index ef8fd1ea..3c32a7fa 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -6,175 +6,128 @@ from flexmock import flexmock from borgmatic.commands import arguments as module -def test_parse_subparser_arguments_consumes_subparser_arguments_before_subparser_name(): - action_namespace = flexmock(foo=True) - subparsers = { - 'action': flexmock(parse_known_args=lambda arguments: (action_namespace, ['action'])), - 'other': flexmock(), - } +def test_get_subaction_parsers_with_no_subactions_returns_empty_result(): + assert module.get_subaction_parsers(flexmock(_subparsers=None)) == {} - arguments, remaining_arguments = module.parse_subparser_arguments( - ('--foo', 'true', 'action'), subparsers + +def test_get_subaction_parsers_with_subactions_returns_one_entry_per_subaction(): + foo_parser = flexmock() + bar_parser = flexmock() + baz_parser = flexmock() + + assert module.get_subaction_parsers( + flexmock( + _subparsers=flexmock( + _group_actions=( + flexmock(choices={'foo': foo_parser, 'bar': bar_parser}), + flexmock(choices={'baz': baz_parser}), + ) + ) + ) + ) == {'foo': foo_parser, 'bar': bar_parser, 'baz': baz_parser} + + +def test_get_subactions_for_actions_with_no_subactions_returns_empty_result(): + assert module.get_subactions_for_actions({'action': flexmock(_subparsers=None)}) == {} + + +def test_get_subactions_for_actions_with_subactions_returns_one_entry_per_action(): + assert module.get_subactions_for_actions( + { + 'action': flexmock( + _subparsers=flexmock( + _group_actions=( + flexmock(choices={'foo': flexmock(), 'bar': flexmock()}), + flexmock(choices={'baz': flexmock()}), + ) + ) + ), + 'other': flexmock( + _subparsers=flexmock(_group_actions=(flexmock(choices={'quux': flexmock()}),)) + ), + } + ) == {'action': ('foo', 'bar', 'baz'), 'other': ('quux',)} + + +def test_omit_values_colliding_with_action_names_drops_action_names_that_have__been_parsed_as_values(): + assert module.omit_values_colliding_with_action_names( + ('check', '--only', 'extract', '--some-list', 'borg'), + {'check': flexmock(only='extract', some_list=['borg'])}, + ) == ('check', '--only', '--some-list') + + +def test_parse_and_record_action_arguments_without_action_name_leaves_arguments_untouched(): + unparsed_arguments = ('--foo', '--bar') + flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( + unparsed_arguments ) - assert arguments == {'action': action_namespace} - assert remaining_arguments == [] - - -def test_parse_subparser_arguments_consumes_subparser_arguments_after_subparser_name(): - action_namespace = flexmock(foo=True) - subparsers = { - 'action': flexmock(parse_known_args=lambda arguments: (action_namespace, ['action'])), - 'other': flexmock(), - } - - arguments, remaining_arguments = module.parse_subparser_arguments( - ('action', '--foo', 'true'), subparsers + assert ( + module.parse_and_record_action_arguments( + unparsed_arguments, flexmock(), flexmock(), 'action' + ) + == unparsed_arguments ) - assert arguments == {'action': action_namespace} - assert remaining_arguments == [] - -def test_parse_subparser_arguments_consumes_subparser_arguments_with_alias(): - action_namespace = flexmock(foo=True) - action_subparser = flexmock(parse_known_args=lambda arguments: (action_namespace, ['action'])) - subparsers = { - 'action': action_subparser, - '-a': action_subparser, - 'other': flexmock(), - '-o': flexmock(), - } - flexmock(module).SUBPARSER_ALIASES = {'action': ['-a'], 'other': ['-o']} - - arguments, remaining_arguments = module.parse_subparser_arguments( - ('-a', '--foo', 'true'), subparsers +def test_parse_and_record_action_arguments_updates_parsed_arguments_and_returns_remaining(): + unparsed_arguments = ('action', '--foo', '--bar', '--verbosity', '1') + other_parsed_arguments = flexmock() + parsed_arguments = {'other': other_parsed_arguments} + action_parsed_arguments = flexmock() + flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( + unparsed_arguments + ) + action_parser = flexmock() + flexmock(action_parser).should_receive('parse_known_args').and_return( + action_parsed_arguments, ('action', '--verbosity', '1') ) - assert arguments == {'action': action_namespace} - assert remaining_arguments == [] + assert module.parse_and_record_action_arguments( + unparsed_arguments, parsed_arguments, action_parser, 'action' + ) == ('--verbosity', '1') + assert parsed_arguments == {'other': other_parsed_arguments, 'action': action_parsed_arguments} -def test_parse_subparser_arguments_consumes_multiple_subparser_arguments(): - action_namespace = flexmock(foo=True) - other_namespace = flexmock(bar=3) - subparsers = { - 'action': flexmock( - parse_known_args=lambda arguments: (action_namespace, ['action', '--bar', '3']) - ), - 'other': flexmock(parse_known_args=lambda arguments: (other_namespace, [])), - } - - arguments, remaining_arguments = module.parse_subparser_arguments( - ('action', '--foo', 'true', 'other', '--bar', '3'), subparsers +def test_parse_and_record_action_arguments_with_alias_updates_canonical_parsed_arguments(): + unparsed_arguments = ('action', '--foo', '--bar', '--verbosity', '1') + other_parsed_arguments = flexmock() + parsed_arguments = {'other': other_parsed_arguments} + action_parsed_arguments = flexmock() + flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( + unparsed_arguments + ) + action_parser = flexmock() + flexmock(action_parser).should_receive('parse_known_args').and_return( + action_parsed_arguments, ('action', '--verbosity', '1') ) - assert arguments == {'action': action_namespace, 'other': other_namespace} - assert remaining_arguments == [] + assert module.parse_and_record_action_arguments( + unparsed_arguments, parsed_arguments, action_parser, 'action', canonical_name='doit' + ) == ('--verbosity', '1') + assert parsed_arguments == {'other': other_parsed_arguments, 'doit': action_parsed_arguments} -def test_parse_subparser_arguments_respects_command_line_action_ordering(): - other_namespace = flexmock() - action_namespace = flexmock(foo=True) - subparsers = { - 'action': flexmock( - parse_known_args=lambda arguments: (action_namespace, ['action', '--foo', 'true']) - ), - 'other': flexmock(parse_known_args=lambda arguments: (other_namespace, ['other'])), - } - - arguments, remaining_arguments = module.parse_subparser_arguments( - ('other', '--foo', 'true', 'action'), subparsers +def test_parse_and_record_action_arguments_with_borg_action_consumes_arguments_after_action_name(): + unparsed_arguments = ('--verbosity', '1', 'borg', 'list') + parsed_arguments = {} + borg_parsed_arguments = flexmock(options=flexmock()) + flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( + unparsed_arguments + ) + borg_parser = flexmock() + flexmock(borg_parser).should_receive('parse_known_args').and_return( + borg_parsed_arguments, ('--verbosity', '1', 'borg', 'list') ) - assert arguments == collections.OrderedDict( - [('other', other_namespace), ('action', action_namespace)] - ) - assert remaining_arguments == [] - - -def test_parse_subparser_arguments_applies_default_subparsers(): - prune_namespace = flexmock() - compact_namespace = flexmock() - create_namespace = flexmock(progress=True) - check_namespace = flexmock() - subparsers = { - 'prune': flexmock( - parse_known_args=lambda arguments: (prune_namespace, ['prune', '--progress']) - ), - 'compact': flexmock(parse_known_args=lambda arguments: (compact_namespace, [])), - 'create': flexmock(parse_known_args=lambda arguments: (create_namespace, [])), - 'check': flexmock(parse_known_args=lambda arguments: (check_namespace, [])), - 'other': flexmock(), - } - - arguments, remaining_arguments = module.parse_subparser_arguments(('--progress'), subparsers) - - assert arguments == { - 'prune': prune_namespace, - 'compact': compact_namespace, - 'create': create_namespace, - 'check': check_namespace, - } - assert remaining_arguments == [] - - -def test_parse_subparser_arguments_passes_through_unknown_arguments_before_subparser_name(): - action_namespace = flexmock() - subparsers = { - 'action': flexmock( - parse_known_args=lambda arguments: (action_namespace, ['action', '--verbosity', 'lots']) - ), - 'other': flexmock(), - } - - arguments, remaining_arguments = module.parse_subparser_arguments( - ('--verbosity', 'lots', 'action'), subparsers - ) - - assert arguments == {'action': action_namespace} - assert remaining_arguments == ['--verbosity', 'lots'] - - -def test_parse_subparser_arguments_passes_through_unknown_arguments_after_subparser_name(): - action_namespace = flexmock() - subparsers = { - 'action': flexmock( - parse_known_args=lambda arguments: (action_namespace, ['action', '--verbosity', 'lots']) - ), - 'other': flexmock(), - } - - arguments, remaining_arguments = module.parse_subparser_arguments( - ('action', '--verbosity', 'lots'), subparsers - ) - - assert arguments == {'action': action_namespace} - assert remaining_arguments == ['--verbosity', 'lots'] - - -def test_parse_subparser_arguments_parses_borg_options_and_skips_other_subparsers(): - action_namespace = flexmock(options=[]) - subparsers = { - 'borg': flexmock(parse_known_args=lambda arguments: (action_namespace, ['borg', 'list'])), - 'list': flexmock(), - } - - arguments, remaining_arguments = module.parse_subparser_arguments(('borg', 'list'), subparsers) - - assert arguments == {'borg': action_namespace} - assert arguments['borg'].options == ['list'] - assert remaining_arguments == [] - - -def test_parse_subparser_arguments_raises_error_when_no_subparser_is_specified(): - action_namespace = flexmock(options=[]) - subparsers = { - 'config': flexmock(parse_known_args=lambda arguments: (action_namespace, ['config'])), - } - - with pytest.raises(ValueError): - module.parse_subparser_arguments(('config',), subparsers) + assert module.parse_and_record_action_arguments( + unparsed_arguments, + parsed_arguments, + borg_parser, + 'borg', + ) == ('--verbosity', '1') + assert parsed_arguments == {'borg': borg_parsed_arguments} + assert borg_parsed_arguments.options == ('list',) @pytest.mark.parametrize( @@ -187,9 +140,7 @@ def test_parse_subparser_arguments_raises_error_when_no_subparser_is_specified() ('prune', 'check', 'list', '--test-flag'), ('prune', 'check', 'extract', '--test-flag'), ), - [ - '--test-flag', - ], + ('--test-flag',), ), ( ( @@ -198,12 +149,241 @@ def test_parse_subparser_arguments_raises_error_when_no_subparser_is_specified() ('prune', 'check', 'list'), ('prune', 'check', 'extract'), ), - [], + (), ), - ((), []), + ((), ()), ], ) -def test_get_unparsable_arguments_returns_remaining_arguments_that_no_subparser_can_parse( +def test_get_unparsable_arguments_returns_remaining_arguments_that_no_action_can_parse( arguments, expected ): assert module.get_unparsable_arguments(arguments) == expected + + +def test_parse_arguments_for_actions_consumes_action_arguments_before_action_name(): + action_namespace = flexmock(foo=True) + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {action: action_namespace} + ) + ).and_return(()) + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + flexmock(module).should_receive('get_unparsable_arguments').and_return(()) + action_parsers = {'action': flexmock(), 'other': flexmock()} + + arguments, remaining_arguments = module.parse_arguments_for_actions( + ('--foo', 'true', 'action'), action_parsers + ) + + assert arguments == {'action': action_namespace} + assert remaining_arguments == () + + +def test_parse_arguments_for_actions_consumes_action_arguments_after_action_name(): + action_namespace = flexmock(foo=True) + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {action: action_namespace} + ) + ).and_return(()) + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + flexmock(module).should_receive('get_unparsable_arguments').and_return(()) + action_parsers = {'action': flexmock(), 'other': flexmock()} + + arguments, remaining_arguments = module.parse_arguments_for_actions( + ('action', '--foo', 'true'), action_parsers + ) + + assert arguments == {'action': action_namespace} + assert remaining_arguments == () + + +def test_parse_arguments_for_actions_consumes_action_arguments_with_alias(): + action_namespace = flexmock(foo=True) + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {canonical or action: action_namespace} + ) + ).and_return(()) + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + flexmock(module).should_receive('get_unparsable_arguments').and_return(()) + action_parsers = { + 'action': flexmock(), + '-a': flexmock(), + 'other': flexmock(), + '-o': flexmock(), + } + flexmock(module).ACTION_ALIASES = {'action': ['-a'], 'other': ['-o']} + + arguments, remaining_arguments = module.parse_arguments_for_actions( + ('-a', '--foo', 'true'), action_parsers + ) + + assert arguments == {'action': action_namespace} + assert remaining_arguments == () + + +def test_parse_arguments_for_actions_consumes_multiple_action_arguments(): + action_namespace = flexmock(foo=True) + other_namespace = flexmock(bar=3) + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {action: action_namespace if action == 'action' else other_namespace} + ) + ).and_return(('other', '--bar', '3')).and_return('action', '--foo', 'true') + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + flexmock(module).should_receive('get_unparsable_arguments').and_return(()) + action_parsers = { + 'action': flexmock(), + 'other': flexmock(), + } + + arguments, remaining_arguments = module.parse_arguments_for_actions( + ('action', '--foo', 'true', 'other', '--bar', '3'), action_parsers + ) + + assert arguments == {'action': action_namespace, 'other': other_namespace} + assert remaining_arguments == () + + +def test_parse_arguments_for_actions_respects_command_line_action_ordering(): + other_namespace = flexmock() + action_namespace = flexmock(foo=True) + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {action: other_namespace if action == 'other' else action_namespace} + ) + ).and_return(('action',)).and_return(('other', '--foo', 'true')) + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + flexmock(module).should_receive('get_unparsable_arguments').and_return(()) + action_parsers = { + 'action': flexmock(), + 'other': flexmock(), + } + + arguments, remaining_arguments = module.parse_arguments_for_actions( + ('other', '--foo', 'true', 'action'), action_parsers + ) + + assert arguments == collections.OrderedDict( + [('other', other_namespace), ('action', action_namespace)] + ) + assert remaining_arguments == () + + +def test_parse_arguments_for_actions_applies_default_action_parsers(): + namespaces = { + 'prune': flexmock(), + 'compact': flexmock(), + 'create': flexmock(progress=True), + 'check': flexmock(), + } + + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {action: namespaces.get(action)} + ) + ).and_return(()) + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + flexmock(module).should_receive('get_unparsable_arguments').and_return(()) + action_parsers = { + 'prune': flexmock(), + 'compact': flexmock(), + 'create': flexmock(), + 'check': flexmock(), + 'other': flexmock(), + } + + arguments, remaining_arguments = module.parse_arguments_for_actions( + ('--progress'), action_parsers + ) + + assert arguments == namespaces + assert remaining_arguments == () + + +def test_parse_arguments_for_actions_passes_through_unknown_arguments_before_action_name(): + action_namespace = flexmock() + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {action: action_namespace} + ) + ).and_return(('--verbosity', 'lots')) + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + flexmock(module).should_receive('get_unparsable_arguments').and_return(('--verbosity', 'lots')) + action_parsers = { + 'action': flexmock(), + 'other': flexmock(), + } + + arguments, remaining_arguments = module.parse_arguments_for_actions( + ('--verbosity', 'lots', 'action'), action_parsers + ) + + assert arguments == {'action': action_namespace} + assert remaining_arguments == ('--verbosity', 'lots') + + +def test_parse_arguments_for_actions_passes_through_unknown_arguments_after_action_name(): + action_namespace = flexmock() + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {action: action_namespace} + ) + ).and_return(('--verbosity', 'lots')) + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + flexmock(module).should_receive('get_unparsable_arguments').and_return(('--verbosity', 'lots')) + action_parsers = { + 'action': flexmock(), + 'other': flexmock(), + } + + arguments, remaining_arguments = module.parse_arguments_for_actions( + ('action', '--verbosity', 'lots'), action_parsers + ) + + assert arguments == {'action': action_namespace} + assert remaining_arguments == ('--verbosity', 'lots') + + +def test_parse_arguments_for_actions_with_borg_action_skips_other_action_parsers(): + action_namespace = flexmock(options=[]) + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {action: action_namespace} + ) + ).and_return(()) + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + flexmock(module).should_receive('get_unparsable_arguments').and_return(()) + action_parsers = { + 'borg': flexmock(), + 'list': flexmock(), + } + + arguments, remaining_arguments = module.parse_arguments_for_actions( + ('borg', 'list'), action_parsers + ) + + assert arguments == {'borg': action_namespace} + assert remaining_arguments == () + + +def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): + flexmock(module).should_receive('get_subaction_parsers').and_return({'bootstrap': [flexmock()]}) + flexmock(module).should_receive('parse_and_record_action_arguments').and_return(flexmock()) + flexmock(module).should_receive('get_subactions_for_actions').and_return( + {'config': ['bootstrap']} + ) + action_parsers = {'config': flexmock()} + + with pytest.raises(ValueError): + module.parse_arguments_for_actions(('config',), action_parsers) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 60fb7568..5ac06326 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1013,7 +1013,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_bootstrap( logs = tuple( module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) ) - assert {log.levelno for log in logs} == {logging.INFO} + assert {log.levelno for log in logs} == {logging.ANSWER} def test_collect_configuration_run_summary_logs_error_on_bootstrap_failure(): From 6098005f5d01c322d99a9dd6842f8aa7b5a1d9bb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 19 Jun 2023 23:07:57 -0700 Subject: [PATCH 282/344] Fix an error when "data" check time files are accessed without getting upgraded first (#711, #713). --- NEWS | 2 ++ borgmatic/borg/check.py | 2 +- tests/unit/borg/test_check.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 50391a87..8f3ffc2c 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,8 @@ * #697, #712: Extract borgmatic configuration from backup via "bootstrap" action—even when borgmatic has no configuration yet! * #669: Add sample systemd user service for running borgmatic as a non-root user. + * #711, #713: Fix an error when "data" check time files are accessed without getting upgraded + first. 1.7.14 * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file, diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 0c417aca..0e54a2cd 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -342,7 +342,7 @@ def upgrade_check_times(location_config, borg_repository_id): temporary_path = f'{old_path}.temp' if not os.path.isfile(old_path) and not os.path.isfile(temporary_path): - return + continue logger.debug(f'Upgrading archives check time from {old_path} to {new_path}') diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index a1044ba4..79201166 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -544,6 +544,34 @@ def test_upgrade_check_times_renames_old_check_paths_to_all(): module.upgrade_check_times(flexmock(), flexmock()) +def test_upgrade_check_times_renames_data_check_paths_when_archives_paths_are_already_upgraded(): + base_path = '~/.borgmatic/checks/1234' + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'archives', 'all' + ).and_return(f'{base_path}/archives/all') + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'data', 'all' + ).and_return(f'{base_path}/data/all') + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/archives.temp' + ).and_return(False) + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return( + True + ) + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/data', f'{base_path}/data.temp' + ).once() + flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/data').once() + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/data.temp', f'{base_path}/data/all' + ).once() + + module.upgrade_check_times(flexmock(), flexmock()) + + def test_upgrade_check_times_skips_missing_check_paths(): flexmock(module).should_receive('make_check_time_path').and_return( '~/.borgmatic/checks/1234/archives/all' From b10aee3070cbf1ec1af1107c4a57ea2e78333d54 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 19 Jun 2023 23:17:59 -0700 Subject: [PATCH 283/344] Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style configuration (#529). --- NEWS | 2 + borgmatic/commands/convert_config.py | 102 -------------- borgmatic/config/convert.py | 95 ------------- docs/how-to/upgrade.md | 26 +--- setup.py | 1 - .../commands/test_convert_config.py | 50 ------- tests/unit/config/test_convert.py | 126 ------------------ 7 files changed, 7 insertions(+), 395 deletions(-) delete mode 100644 borgmatic/commands/convert_config.py delete mode 100644 borgmatic/config/convert.py delete mode 100644 tests/integration/commands/test_convert_config.py delete mode 100644 tests/unit/config/test_convert.py diff --git a/NEWS b/NEWS index 8f3ffc2c..099b81d2 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ 1.7.15.dev0 * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors. + * #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style + configuration. * #697, #712: Extract borgmatic configuration from backup via "bootstrap" action—even when borgmatic has no configuration yet! * #669: Add sample systemd user service for running borgmatic as a non-root user. diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py deleted file mode 100644 index 64a89486..00000000 --- a/borgmatic/commands/convert_config.py +++ /dev/null @@ -1,102 +0,0 @@ -import os -import sys -import textwrap -from argparse import ArgumentParser - -from ruamel import yaml - -from borgmatic.config import convert, generate, legacy, validate - -DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config' -DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' -DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' - - -def parse_arguments(*arguments): - ''' - Given command-line arguments with which this script was invoked, parse the arguments and return - them as an ArgumentParser instance. - ''' - parser = ArgumentParser( - description=''' - Convert legacy INI-style borgmatic configuration and excludes files to a single YAML - configuration file. Note that this replaces any comments from the source files. - ''' - ) - parser.add_argument( - '-s', - '--source-config', - dest='source_config_filename', - default=DEFAULT_SOURCE_CONFIG_FILENAME, - help=f'Source INI-style configuration filename. Default: {DEFAULT_SOURCE_CONFIG_FILENAME}', - ) - parser.add_argument( - '-e', - '--source-excludes', - dest='source_excludes_filename', - default=DEFAULT_SOURCE_EXCLUDES_FILENAME - if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) - else None, - help='Excludes filename', - ) - parser.add_argument( - '-d', - '--destination-config', - dest='destination_config_filename', - default=DEFAULT_DESTINATION_CONFIG_FILENAME, - help=f'Destination YAML configuration filename. Default: {DEFAULT_DESTINATION_CONFIG_FILENAME}', - ) - - return parser.parse_args(arguments) - - -TEXT_WRAP_CHARACTERS = 80 - - -def display_result(args): # pragma: no cover - result_lines = textwrap.wrap( - f'Your borgmatic configuration has been upgraded. Please review the result in {args.destination_config_filename}.', - TEXT_WRAP_CHARACTERS, - ) - - excludes_phrase = ( - f' and {args.source_excludes_filename}' if args.source_excludes_filename else '' - ) - delete_lines = textwrap.wrap( - f'Once you are satisfied, you can safely delete {args.source_config_filename}{excludes_phrase}.', - TEXT_WRAP_CHARACTERS, - ) - - print('\n'.join(result_lines)) - print() - print('\n'.join(delete_lines)) - - -def main(): # pragma: no cover - try: - args = parse_arguments(*sys.argv[1:]) - schema = yaml.round_trip_load(open(validate.schema_filename()).read()) - source_config = legacy.parse_configuration( - args.source_config_filename, legacy.CONFIG_FORMAT - ) - source_config_file_mode = os.stat(args.source_config_filename).st_mode - source_excludes = ( - open(args.source_excludes_filename).read().splitlines() - if args.source_excludes_filename - else [] - ) - - destination_config = convert.convert_legacy_parsed_config( - source_config, source_excludes, schema - ) - - generate.write_configuration( - args.destination_config_filename, - generate.render_configuration(destination_config), - mode=source_config_file_mode, - ) - - display_result(args) - except (ValueError, OSError) as error: - print(error, file=sys.stderr) - sys.exit(1) diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py deleted file mode 100644 index 093ad0c1..00000000 --- a/borgmatic/config/convert.py +++ /dev/null @@ -1,95 +0,0 @@ -import os - -from ruamel import yaml - -from borgmatic.config import generate - - -def _convert_section(source_section_config, section_schema): - ''' - Given a legacy Parsed_config instance for a single section, convert it to its corresponding - yaml.comments.CommentedMap representation in preparation for actual serialization to YAML. - - Where integer types exist in the given section schema, convert their values to integers. - ''' - destination_section_config = yaml.comments.CommentedMap( - [ - ( - option_name, - int(option_value) - if section_schema['properties'].get(option_name, {}).get('type') == 'integer' - else option_value, - ) - for option_name, option_value in source_section_config.items() - ] - ) - - return destination_section_config - - -def convert_legacy_parsed_config(source_config, source_excludes, schema): - ''' - Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude - patterns, convert them to a corresponding yaml.comments.CommentedMap representation in - preparation for serialization to a single YAML config file. - - Additionally, use the given schema as a source of helpful comments to include within the - returned CommentedMap. - ''' - destination_config = yaml.comments.CommentedMap( - [ - (section_name, _convert_section(section_config, schema['properties'][section_name])) - for section_name, section_config in source_config._asdict().items() - ] - ) - - # Split space-separated values into actual lists, make "repository" into a list, and merge in - # excludes. - location = destination_config['location'] - location['source_directories'] = source_config.location['source_directories'].split(' ') - location['repositories'] = [location.pop('repository')] - location['exclude_patterns'] = source_excludes - - if source_config.consistency.get('checks'): - destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') - - # Add comments to each section, and then add comments to the fields in each section. - generate.add_comments_to_configuration_object(destination_config, schema) - - for section_name, section_config in destination_config.items(): - generate.add_comments_to_configuration_object( - section_config, schema['properties'][section_name], indent=generate.INDENT - ) - - return destination_config - - -class Legacy_configuration_not_upgraded(FileNotFoundError): - def __init__(self): - super(Legacy_configuration_not_upgraded, self).__init__( - '''borgmatic changed its configuration file format in version 1.1.0 from INI-style -to YAML. This better supports validation, and has a more natural way to express -lists of values. To upgrade your existing configuration, run: - - sudo upgrade-borgmatic-config - -That will generate a new YAML configuration file at /etc/borgmatic/config.yaml -(by default) using the values from both your existing configuration and excludes -files. The new version of borgmatic will consume the YAML configuration file -instead of the old one.''' - ) - - -def guard_configuration_upgraded(source_config_filename, destination_config_filenames): - ''' - If legacy source configuration exists but no destination upgraded configs do, raise - Legacy_configuration_not_upgraded. - - The idea is that we want to alert the user about upgrading their config if they haven't already. - ''' - destination_config_exists = any( - os.path.exists(filename) for filename in destination_config_filenames - ) - - if os.path.exists(source_config_filename) and not destination_config_exists: - raise Legacy_configuration_not_upgraded() diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index 69b5f5b8..27778f13 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -61,21 +61,22 @@ and, if desired, replace your original configuration file with it. borgmatic changed its configuration file format in version 1.1.0 from INI-style to YAML. This better supports validation, and has a more natural way to express lists of values. To upgrade your existing configuration, first -upgrade to the new version of borgmatic. +upgrade to the last version of borgmatic to support converting configuration: +borgmatic 1.7.14. As of version 1.1.0, borgmatic no longer supports Python 2. If you were already running borgmatic with Python 3, then you can upgrade borgmatic in-place: ```bash -sudo pip3 install --user --upgrade borgmatic +sudo pip3 install --user --upgrade borgmatic==1.7.14 ``` But if you were running borgmatic with Python 2, uninstall and reinstall instead: ```bash sudo pip uninstall borgmatic -sudo pip3 install --user borgmatic +sudo pip3 install --user borgmatic==1.7.14 ``` The pip binary names for different versions of Python can differ, so the above @@ -93,29 +94,12 @@ That will generate a new YAML configuration file at /etc/borgmatic/config.yaml excludes files. The new version of borgmatic will consume the YAML configuration file instead of the old one. - -### Upgrading from atticmatic - -You can ignore this section if you're not an atticmatic user (the former name -of borgmatic). - -borgmatic only supports Borg now and no longer supports Attic. So if you're -an Attic user, consider switching to Borg. See the [Borg upgrade -command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade) -for more information. Then, follow the instructions above about setting up -your borgmatic configuration files. - -If you were already using Borg with atticmatic, then you can upgrade -from atticmatic to borgmatic by running the following commands: +Now you can upgrade to a newer version of borgmatic: ```bash -sudo pip3 uninstall atticmatic sudo pip3 install --user borgmatic ``` -That's it! borgmatic will continue using your /etc/borgmatic configuration -files. - ## Upgrading Borg diff --git a/setup.py b/setup.py index 8fb11b1f..4fa049e0 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ setup( entry_points={ 'console_scripts': [ 'borgmatic = borgmatic.commands.borgmatic:main', - 'upgrade-borgmatic-config = borgmatic.commands.convert_config:main', 'generate-borgmatic-config = borgmatic.commands.generate_config:main', 'validate-borgmatic-config = borgmatic.commands.validate_config:main', ] diff --git a/tests/integration/commands/test_convert_config.py b/tests/integration/commands/test_convert_config.py deleted file mode 100644 index db227108..00000000 --- a/tests/integration/commands/test_convert_config.py +++ /dev/null @@ -1,50 +0,0 @@ -import os - -import pytest -from flexmock import flexmock - -from borgmatic.commands import convert_config as module - - -def test_parse_arguments_with_no_arguments_uses_defaults(): - flexmock(os.path).should_receive('exists').and_return(True) - - parser = module.parse_arguments() - - assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME - assert parser.source_excludes_filename == module.DEFAULT_SOURCE_EXCLUDES_FILENAME - assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME - - -def test_parse_arguments_with_filename_arguments_overrides_defaults(): - flexmock(os.path).should_receive('exists').and_return(True) - - parser = module.parse_arguments( - '--source-config', - 'config', - '--source-excludes', - 'excludes', - '--destination-config', - 'config.yaml', - ) - - assert parser.source_config_filename == 'config' - assert parser.source_excludes_filename == 'excludes' - assert parser.destination_config_filename == 'config.yaml' - - -def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none(): - flexmock(os.path).should_receive('exists').and_return(False) - - parser = module.parse_arguments() - - assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME - assert parser.source_excludes_filename is None - assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME - - -def test_parse_arguments_with_invalid_arguments_exits(): - flexmock(os.path).should_receive('exists').and_return(True) - - with pytest.raises(SystemExit): - module.parse_arguments('--posix-me-harder') diff --git a/tests/unit/config/test_convert.py b/tests/unit/config/test_convert.py deleted file mode 100644 index 8ce1cdc6..00000000 --- a/tests/unit/config/test_convert.py +++ /dev/null @@ -1,126 +0,0 @@ -import os -from collections import OrderedDict, defaultdict, namedtuple - -import pytest -from flexmock import flexmock - -from borgmatic.config import convert as module - -Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency')) - - -def test_convert_section_generates_integer_value_for_integer_type_in_schema(): - flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) - source_section_config = OrderedDict([('check_last', '3')]) - section_schema = {'type': 'object', 'properties': {'check_last': {'type': 'integer'}}} - - destination_config = module._convert_section(source_section_config, section_schema) - - assert destination_config == OrderedDict([('check_last', 3)]) - - -def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): - flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) - flexmock(module.generate).should_receive('add_comments_to_configuration_object') - source_config = Parsed_config( - location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]), - storage=OrderedDict([('encryption_passphrase', 'supersecret')]), - retention=OrderedDict([('keep_daily', 7)]), - consistency=OrderedDict([('checks', 'repository')]), - ) - source_excludes = ['/var'] - schema = { - 'type': 'object', - 'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}), - } - - destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema) - - assert destination_config == OrderedDict( - [ - ( - 'location', - OrderedDict( - [ - ('source_directories', ['/home']), - ('repositories', ['hostname.borg']), - ('exclude_patterns', ['/var']), - ] - ), - ), - ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])), - ('retention', OrderedDict([('keep_daily', 7)])), - ('consistency', OrderedDict([('checks', ['repository'])])), - ] - ) - - -def test_convert_legacy_parsed_config_splits_space_separated_values(): - flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) - flexmock(module.generate).should_receive('add_comments_to_configuration_object') - source_config = Parsed_config( - location=OrderedDict( - [('source_directories', '/home /etc'), ('repository', 'hostname.borg')] - ), - storage=OrderedDict(), - retention=OrderedDict(), - consistency=OrderedDict([('checks', 'repository archives')]), - ) - source_excludes = ['/var'] - schema = { - 'type': 'object', - 'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}), - } - - destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema) - - assert destination_config == OrderedDict( - [ - ( - 'location', - OrderedDict( - [ - ('source_directories', ['/home', '/etc']), - ('repositories', ['hostname.borg']), - ('exclude_patterns', ['/var']), - ] - ), - ), - ('storage', OrderedDict()), - ('retention', OrderedDict()), - ('consistency', OrderedDict([('checks', ['repository', 'archives'])])), - ] - ) - - -def test_guard_configuration_upgraded_raises_when_only_source_config_present(): - flexmock(os.path).should_receive('exists').with_args('config').and_return(True) - flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) - flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False) - - with pytest.raises(module.Legacy_configuration_not_upgraded): - module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) - - -def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present(): - flexmock(os.path).should_receive('exists').with_args('config').and_return(False) - flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) - flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True) - - module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) - - -def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present(): - flexmock(os.path).should_receive('exists').with_args('config').and_return(True) - flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) - flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True) - - module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) - - -def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present(): - flexmock(os.path).should_receive('exists').with_args('config').and_return(False) - flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) - flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False) - - module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) From 147516ae3f043744daae120a2b1610aa64591411 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 20 Jun 2023 09:41:26 -0700 Subject: [PATCH 284/344] Remove additional upgrade-borgmatic-config code (#529). --- borgmatic/commands/borgmatic.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index babfba0b..a76dd4f8 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -36,7 +36,7 @@ import borgmatic.commands.completion.fish from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments -from borgmatic.config import checks, collect, convert, validate +from borgmatic.config import checks, collect, validate from borgmatic.hooks import command, dispatch, monitor from borgmatic.logger import DISABLED, add_custom_log_levels, configure_logging, should_do_markup from borgmatic.signals import configure_signals @@ -785,7 +785,6 @@ def main(): # pragma: no cover exit_with_help_link() logger.debug('Ensuring legacy configuration is upgraded') - convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames) summary_logs = parse_logs + list(collect_configuration_run_summary_logs(configs, arguments)) summary_logs_max_level = max(log.levelno for log in summary_logs) From 87c6e5b349710535b01e607b6135e22509482374 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Wed, 21 Jun 2023 00:03:07 +0530 Subject: [PATCH 285/344] make sure restore params in config aren't used when cli args are supplied --- tests/unit/hooks/test_mongodb.py | 4 ++++ tests/unit/hooks/test_mysql.py | 12 +++++++++++- tests/unit/hooks/test_postgresql.py | 4 ++++ tests/unit/hooks/test_sqlite.py | 4 +++- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 14f7cafb..5ac8ce96 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -304,6 +304,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for 'username': 'mongo', 'password': 'trustsome1', 'authentication_database': 'admin', + 'restore_hostname': 'restorehost', + 'restore_port': 'restoreport', + 'restore_username': 'restoreusername', + 'restore_password': 'restorepassword', 'schemas': None, } ] diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index e45fa56e..cdcddf5e 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -519,7 +519,17 @@ def test_restore_database_dump_runs_mysql_with_username_and_password(): def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): - database_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] + database_config = [ + { + 'name': 'foo', + 'username': 'root', + 'password': 'trustsome1', + 'restore_hostname': 'restorehost', + 'restore_port': 'restoreport', + 'restore_username': 'restoreusername', + 'restore_password': 'restorepassword', + } + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('execute_command_with_processes').with_args( diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 5120ad02..e48258ee 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -656,6 +656,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for 'port': 5433, 'username': 'postgres', 'password': 'trustsome1', + 'restore_hostname': 'restorehost', + 'restore_port': 'restoreport', + 'restore_username': 'restoreusername', + 'restore_password': 'restorepassword', 'schemas': None, } ] diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py index 30700e6e..5820713b 100644 --- a/tests/unit/hooks/test_sqlite.py +++ b/tests/unit/hooks/test_sqlite.py @@ -118,7 +118,9 @@ def test_restore_database_dump_restores_database(): def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): - database_config = [{'path': '/path/to/database', 'name': 'database'}] + database_config = [ + {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} + ] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('execute_command_with_processes').with_args( From 248f82d6f6ac48504447f5da9cc0f1643a6a7100 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 21 Jun 2023 10:41:32 -0700 Subject: [PATCH 286/344] Fix for another subaction argument-parsing edge case (#712). --- borgmatic/commands/arguments.py | 16 ++++++++++------ tests/integration/commands/test_arguments.py | 8 +++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 2f978a3f..02a0d119 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -161,17 +161,21 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers): subaction_parsers = get_subaction_parsers(action_parser) - # Parse with subaction parsers, if any. + # But first parse with subaction parsers, if any. if subaction_parsers: subactions_parsed = False for subaction_name, subaction_parser in subaction_parsers.items(): remaining_action_arguments.append( - parse_and_record_action_arguments( - unparsed_arguments, - arguments, - subaction_parser, - subaction_name, + tuple( + argument + for argument in parse_and_record_action_arguments( + unparsed_arguments, + arguments, + subaction_parser, + subaction_name, + ) + if argument != action_name ) ) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index b2b03849..ea5891d1 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -587,7 +587,13 @@ def test_parse_arguments_config_with_subaction_and_required_flags_does_not_raise module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg') -def test_parse_arguments_config_with_subaction_and_global_flags_does_not_raise(capsys): +def test_parse_arguments_config_with_subaction_and_global_flags_at_start_does_not_raise(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments('--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg') + + +def test_parse_arguments_config_with_subaction_and_global_flags_at_end_does_not_raise(capsys): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1') From 803fc25848b9218c0b84d91cc6d9fd1404cb5e8f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 21 Jun 2023 10:47:53 -0700 Subject: [PATCH 287/344] Add a test for another edge case (#712). --- tests/integration/commands/test_arguments.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index ea5891d1..47ec72fa 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -561,7 +561,7 @@ def test_parse_arguments_config_with_help_shows_config_help(capsys): assert 'config sub-actions:' in captured.out -def test_parse_arguments_config_with_subaction_but_missing_flags_errors(capsys): +def test_parse_arguments_config_with_subaction_but_missing_flags_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: @@ -581,19 +581,27 @@ def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(cap assert 'config bootstrap arguments:' in captured.out -def test_parse_arguments_config_with_subaction_and_required_flags_does_not_raise(capsys): +def test_parse_arguments_config_with_subaction_and_required_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg') -def test_parse_arguments_config_with_subaction_and_global_flags_at_start_does_not_raise(capsys): +def test_parse_arguments_config_with_subaction_and_global_flags_at_start_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments('--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg') -def test_parse_arguments_config_with_subaction_and_global_flags_at_end_does_not_raise(capsys): +def test_parse_arguments_config_with_subaction_and_global_flags_at_end_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1') + + +def test_parse_arguments_config_with_subaction_and_explicit_config_file_does_not_raise(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + module.parse_arguments( + 'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml' + ) From 1b90da5bf1e0db46bf906f3a4a86f9a0d25e5c0d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 21 Jun 2023 12:19:49 -0700 Subject: [PATCH 288/344] Deprecate generate-borgmatic-config in favor if new "config generate" action (#529). --- NEWS | 5 +- borgmatic/actions/config/generate.py | 39 +++++++ borgmatic/commands/arguments.py | 45 +++++-- borgmatic/commands/borgmatic.py | 67 +++++++++-- borgmatic/commands/completion/fish.py | 2 +- borgmatic/commands/generate_config.py | 68 ++--------- borgmatic/config/generate.py | 5 +- docs/Dockerfile | 2 +- docs/how-to/make-per-application-backups.md | 12 +- docs/how-to/set-up-backups.md | 18 ++- docs/how-to/upgrade.md | 14 ++- .../commands/test_generate_config.py | 26 +---- tests/integration/config/test_generate.py | 17 ++- tests/unit/actions/config/test_bootstrap.py | 1 + tests/unit/actions/config/test_generate.py | 39 +++++++ tests/unit/commands/test_borgmatic.py | 110 ++++++++++++------ 16 files changed, 317 insertions(+), 153 deletions(-) create mode 100644 borgmatic/actions/config/generate.py create mode 100644 tests/unit/actions/config/test_generate.py diff --git a/NEWS b/NEWS index 099b81d2..b6939ffa 100644 --- a/NEWS +++ b/NEWS @@ -2,8 +2,9 @@ * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors. * #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style configuration. - * #697, #712: Extract borgmatic configuration from backup via "bootstrap" action—even when - borgmatic has no configuration yet! + * #529: Deprecate generate-borgmatic-config in favor if new "config generate" action. + * #697, #712: Extract borgmatic configuration from backup via new "config bootstrap" action—even + when borgmatic has no configuration yet! * #669: Add sample systemd user service for running borgmatic as a non-root user. * #711, #713: Fix an error when "data" check time files are accessed without getting upgraded first. diff --git a/borgmatic/actions/config/generate.py b/borgmatic/actions/config/generate.py new file mode 100644 index 00000000..5f430383 --- /dev/null +++ b/borgmatic/actions/config/generate.py @@ -0,0 +1,39 @@ +import logging + +import borgmatic.config.generate +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_generate(generate_arguments, global_arguments): + dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else '' + + logger.answer( + f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}' + ) + + borgmatic.config.generate.generate_sample_configuration( + global_arguments.dry_run, + generate_arguments.source_filename, + generate_arguments.destination_filename, + borgmatic.config.validate.schema_filename(), + overwrite=generate_arguments.overwrite, + ) + + if generate_arguments.source_filename: + logger.answer( + f''' +Merged in the contents of configuration file at: {generate_arguments.source_filename} +To review the changes made, run: + + diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}''' + ) + + logger.answer( + ''' +This includes all available configuration options with example values, the few +required options as indicated. Please edit the file to suit your needs. + +If you ever need help: https://torsion.org/borgmatic/#issues''' + ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 02a0d119..24853e3d 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -695,14 +695,12 @@ def make_parsers(): config_parsers = config_parser.add_subparsers( title='config sub-actions', - description='Valid sub-actions for config', - help='Additional help', ) config_bootstrap_parser = config_parsers.add_parser( 'bootstrap', - help='Extract the config files used to create a borgmatic repository', - description='Extract config files that were used to create a borgmatic repository during the "create" action', + help='Extract the borgmatic config files from a named archive', + description='Extract the borgmatic config files from a named archive', add_help=False, ) config_bootstrap_group = config_bootstrap_parser.add_argument_group( @@ -746,6 +744,36 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + config_generate_parser = config_parsers.add_parser( + 'generate', + help='Generate a sample borgmatic configuration file', + description='Generate a sample borgmatic configuration file', + add_help=False, + ) + config_generate_group = config_generate_parser.add_argument_group('config generate arguments') + config_generate_group.add_argument( + '-s', + '--source', + dest='source_filename', + help='Optional configuration file to merge into the generated configuration, useful for upgrading your configuration', + ) + config_generate_group.add_argument( + '-d', + '--destination', + dest='destination_filename', + default=config_paths[0], + help=f'Destination configuration file, default: {unexpanded_config_paths[0]}', + ) + config_generate_group.add_argument( + '--overwrite', + default=False, + action='store_true', + help='Whether to overwrite any existing destination file, defaults to false', + ) + config_generate_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) + export_tar_parser = action_parsers.add_parser( 'export-tar', aliases=ACTION_ALIASES['export-tar'], @@ -1170,10 +1198,11 @@ def parse_arguments(*unparsed_arguments): unparsed_arguments, action_parsers.choices ) - if 'bootstrap' in arguments.keys() and len(arguments.keys()) > 1: - raise ValueError( - 'The bootstrap action cannot be combined with other actions. Please run it separately.' - ) + for action_name in ('bootstrap', 'generate', 'validate'): + if action_name in arguments.keys() and len(arguments.keys()) > 1: + raise ValueError( + 'The {action_name} action cannot be combined with other actions. Please run it separately.' + ) arguments['global'] = top_level_parser.parse_args(remaining_arguments) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a76dd4f8..e04a785a 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -19,6 +19,7 @@ import borgmatic.actions.break_lock import borgmatic.actions.check import borgmatic.actions.compact import borgmatic.actions.config.bootstrap +import borgmatic.actions.config.generate import borgmatic.actions.create import borgmatic.actions.export_tar import borgmatic.actions.extract @@ -602,19 +603,24 @@ def get_local_path(configs): return next(iter(configs.values())).get('location', {}).get('local_path', 'borg') -def collect_configuration_run_summary_logs(configs, arguments): +def collect_highlander_action_summary_logs(configs, arguments): ''' - Given a dict of configuration filename to corresponding parsed configuration, and parsed + Given a dict of configuration filename to corresponding parsed configuration and parsed command-line arguments as a dict from subparser name to a parsed namespace of arguments, run - each configuration file and yield a series of logging.LogRecord instances containing summary - information about each run. + a highlander action specified in the arguments, if any, and yield a series of logging.LogRecord + instances containing summary information. - As a side effect of running through these configuration files, output their JSON results, if - any, to stdout. + A highlander action is an action that cannot coexist with other actions on the borgmatic + command-line, and borgmatic exits after processing such an action. ''' if 'bootstrap' in arguments: - # No configuration file is needed for bootstrap. - local_borg_version = borg_version.local_borg_version({}, 'borg') + try: + # No configuration file is needed for bootstrap. + local_borg_version = borg_version.local_borg_version({}, 'borg') + except (OSError, CalledProcessError, ValueError) as error: + yield from log_error_records('Error getting local Borg version', error) + return + try: borgmatic.actions.config.bootstrap.run_bootstrap( arguments['bootstrap'], arguments['global'], local_borg_version @@ -622,7 +628,7 @@ def collect_configuration_run_summary_logs(configs, arguments): yield logging.makeLogRecord( dict( levelno=logging.ANSWER, - levelname='INFO', + levelname='ANSWER', msg='Bootstrap successful', ) ) @@ -635,6 +641,38 @@ def collect_configuration_run_summary_logs(configs, arguments): return + if 'generate' in arguments: + try: + borgmatic.actions.config.generate.run_generate( + arguments['generate'], arguments['global'] + ) + yield logging.makeLogRecord( + dict( + levelno=logging.ANSWER, + levelname='ANSWER', + msg='Generate successful', + ) + ) + except ( + CalledProcessError, + ValueError, + OSError, + ) as error: + yield from log_error_records(error) + + return + + +def collect_configuration_run_summary_logs(configs, arguments): + ''' + Given a dict of configuration filename to corresponding parsed configuration and parsed + command-line arguments as a dict from subparser name to a parsed namespace of arguments, run + each configuration file and yield a series of logging.LogRecord instances containing summary + information about each run. + + As a side effect of running through these configuration files, output their JSON results, if + any, to stdout. + ''' # Run cross-file validation checks. repository = None @@ -730,7 +768,7 @@ def exit_with_help_link(): # pragma: no cover sys.exit(1) -def main(): # pragma: no cover +def main(extra_summary_logs=[]): # pragma: no cover configure_signals() try: @@ -786,7 +824,14 @@ def main(): # pragma: no cover logger.debug('Ensuring legacy configuration is upgraded') - summary_logs = parse_logs + list(collect_configuration_run_summary_logs(configs, arguments)) + summary_logs = ( + parse_logs + + ( + list(collect_highlander_action_summary_logs(configs, arguments)) + or list(collect_configuration_run_summary_logs(configs, arguments)) + ) + + extra_summary_logs + ) summary_logs_max_level = max(log.levelno for log in summary_logs) for message in ('', 'summary:'): diff --git a/borgmatic/commands/completion/fish.py b/borgmatic/commands/completion/fish.py index 306de195..599617ce 100644 --- a/borgmatic/commands/completion/fish.py +++ b/borgmatic/commands/completion/fish.py @@ -167,6 +167,6 @@ def fish_completion(): f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}''' for action_name, subparser in subparsers.choices.items() for action in subparser._actions - if 'Deprecated' not in action.help + if 'Deprecated' not in (action.help or ()) ) ) diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py index 78c32f04..f95b3094 100644 --- a/borgmatic/commands/generate_config.py +++ b/borgmatic/commands/generate_config.py @@ -1,63 +1,17 @@ +import logging import sys -from argparse import ArgumentParser -from borgmatic.config import generate, validate - -DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' +import borgmatic.commands.borgmatic -def parse_arguments(*arguments): - ''' - Given command-line arguments with which this script was invoked, parse the arguments and return - them as an ArgumentParser instance. - ''' - parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.') - parser.add_argument( - '-s', - '--source', - dest='source_filename', - help='Optional YAML configuration file to merge into the generated configuration, useful for upgrading your configuration', - ) - parser.add_argument( - '-d', - '--destination', - dest='destination_filename', - default=DEFAULT_DESTINATION_CONFIG_FILENAME, - help=f'Destination YAML configuration file, default: {DEFAULT_DESTINATION_CONFIG_FILENAME}', - ) - parser.add_argument( - '--overwrite', - default=False, - action='store_true', - help='Whether to overwrite any existing destination file, defaults to false', - ) - - return parser.parse_args(arguments) - - -def main(): # pragma: no cover - try: - args = parse_arguments(*sys.argv[1:]) - - generate.generate_sample_configuration( - args.source_filename, - args.destination_filename, - validate.schema_filename(), - overwrite=args.overwrite, +def main(): + warning_log = logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg='generate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config generate" instead.', ) + ) - print(f'Generated a sample configuration file at {args.destination_filename}.') - print() - if args.source_filename: - print(f'Merged in the contents of configuration file at {args.source_filename}.') - print('To review the changes made, run:') - print() - print(f' diff --unified {args.source_filename} {args.destination_filename}') - print() - print('This includes all available configuration options with example values. The few') - print('required options are indicated. Please edit the file to suit your needs.') - print() - print('If you ever need help: https://torsion.org/borgmatic/#issues') - except (ValueError, OSError) as error: - print(error, file=sys.stderr) - sys.exit(1) + sys.argv = ['borgmatic', 'config', 'generate'] + sys.argv[1:] + borgmatic.commands.borgmatic.main([warning_log]) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 081186e3..6ef8e3ae 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -267,7 +267,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi def generate_sample_configuration( - source_filename, destination_filename, schema_filename, overwrite=False + dry_run, source_filename, destination_filename, schema_filename, overwrite=False ): ''' Given an optional source configuration filename, and a required destination configuration @@ -287,6 +287,9 @@ def generate_sample_configuration( _schema_to_sample_configuration(schema), source_config ) + if dry_run: + return + write_configuration( destination_filename, _comment_out_optional_configuration(render_configuration(destination_config)), diff --git a/docs/Dockerfile b/docs/Dockerfile index 750d4e53..4ac1867b 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -4,7 +4,7 @@ COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ - && for action in rcreate transfer create prune compact check extract config "config bootstrap" export-tar mount umount restore rlist list rinfo info break-lock borg; do \ + && for action in rcreate transfer create prune compact check extract config "config bootstrap" "config generate" export-tar mount umount restore rlist list rinfo info break-lock borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic $action --help >> /command-line.txt; done diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index b565d668..1f725491 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -20,18 +20,22 @@ instance, for applications: ```bash sudo mkdir /etc/borgmatic.d -sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml -sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml +sudo borgmatic config generate --destination /etc/borgmatic.d/app1.yaml +sudo borgmatic config generate --destination /etc/borgmatic.d/app2.yaml ``` Or, for repositories: ```bash sudo mkdir /etc/borgmatic.d -sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo1.yaml -sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo2.yaml +sudo borgmatic config generate --destination /etc/borgmatic.d/repo1.yaml +sudo borgmatic config generate --destination /etc/borgmatic.d/repo2.yaml ``` +Prior to version 1.7.15 The +command to generate configuation files was `generate-borgmatic-config` instead +of `borgmatic config generate`. + When you set up multiple configuration files like this, borgmatic will run each one in turn from a single borgmatic invocation. This includes, by default, the traditional `/etc/borgmatic/config.yaml` as well. diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 08aea148..dcae31c0 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -120,16 +120,24 @@ offerings, but do not currently fund borgmatic development or hosting. After you install borgmatic, generate a sample configuration file: +```bash +sudo borgmatic config generate +``` + +Prior to version 1.7.15 +Generate a configuation file with this command instead: + ```bash sudo generate-borgmatic-config ``` -If that command is not found, then it may be installed in a location that's -not in your system `PATH` (see above). Try looking in `~/.local/bin/`. +If neither command is found, then borgmatic may be installed in a location +that's not in your system `PATH` (see above). Try looking in `~/.local/bin/`. -This generates a sample configuration file at `/etc/borgmatic/config.yaml` by -default. If you'd like to use another path, use the `--destination` flag, for -instance: `--destination ~/.config/borgmatic/config.yaml`. +The command generates a sample configuration file at +`/etc/borgmatic/config.yaml` by default. If you'd like to use another path, +use the `--destination` flag, for instance: `--destination +~/.config/borgmatic/config.yaml`. You should edit the configuration file to suit your needs, as the generated values are only representative. All options are optional except where diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index 27778f13..3cd12601 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -29,29 +29,33 @@ configuration options. This is completely optional. If you prefer, you can add new configuration options manually. If you do want to upgrade your configuration file to include new options, use -the `generate-borgmatic-config` script with its optional `--source` flag that +the `borgmatic config generate` action with its optional `--source` flag that takes the path to your original configuration file. If provided with this -path, `generate-borgmatic-config` merges your original configuration into the +path, `borgmatic config generate` merges your original configuration into the generated configuration file, so you get all the newest options and comments. Here's an example: ```bash -generate-borgmatic-config --source config.yaml --destination config-new.yaml +borgmatic config generate --source config.yaml --destination config-new.yaml ``` +Prior to version 1.7.15 The +command to generate configuation files was `generate-borgmatic-config` instead +of `borgmatic config generate`. + New options start as commented out, so you can edit the file and decide whether you want to use each one. There are a few caveats to this process. First, when generating the new -configuration file, `generate-borgmatic-config` replaces any comments you've +configuration file, `borgmatic config generate` replaces any comments you've written in your original configuration file with the newest generated comments. Second, the script adds back any options you had originally deleted, although it does so with the options commented out. And finally, any YAML includes you've used in the source configuration get flattened out into a single generated file. -As a safety measure, `generate-borgmatic-config` refuses to modify +As a safety measure, `borgmatic config generate` refuses to modify configuration files in-place. So it's up to you to review the generated file and, if desired, replace your original configuration file with it. diff --git a/tests/integration/commands/test_generate_config.py b/tests/integration/commands/test_generate_config.py index 4cd54429..a292faee 100644 --- a/tests/integration/commands/test_generate_config.py +++ b/tests/integration/commands/test_generate_config.py @@ -1,25 +1,9 @@ +from flexmock import flexmock + from borgmatic.commands import generate_config as module -def test_parse_arguments_with_no_arguments_uses_default_destination(): - parser = module.parse_arguments() +def test_main_does_not_raise(): + flexmock(module.borgmatic.commands.borgmatic).should_receive('main') - assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME - - -def test_parse_arguments_with_destination_argument_overrides_default(): - parser = module.parse_arguments('--destination', 'config.yaml') - - assert parser.destination_filename == 'config.yaml' - - -def test_parse_arguments_parses_source(): - parser = module.parse_arguments('--source', 'source.yaml', '--destination', 'config.yaml') - - assert parser.source_filename == 'source.yaml' - - -def test_parse_arguments_parses_overwrite(): - parser = module.parse_arguments('--destination', 'config.yaml', '--overwrite') - - assert parser.overwrite + module.main() diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 637bb771..cf4b3945 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -210,7 +210,7 @@ def test_generate_sample_configuration_does_not_raise(): flexmock(module).should_receive('_comment_out_optional_configuration') flexmock(module).should_receive('write_configuration') - module.generate_sample_configuration(None, 'dest.yaml', 'schema.yaml') + module.generate_sample_configuration(False, None, 'dest.yaml', 'schema.yaml') def test_generate_sample_configuration_with_source_filename_does_not_raise(): @@ -225,4 +225,17 @@ def test_generate_sample_configuration_with_source_filename_does_not_raise(): flexmock(module).should_receive('_comment_out_optional_configuration') flexmock(module).should_receive('write_configuration') - module.generate_sample_configuration('source.yaml', 'dest.yaml', 'schema.yaml') + module.generate_sample_configuration(False, 'source.yaml', 'dest.yaml', 'schema.yaml') + + +def test_generate_sample_configuration_with_dry_run_does_not_write_file(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('schema.yaml').and_return('') + flexmock(module.yaml).should_receive('round_trip_load') + flexmock(module).should_receive('_schema_to_sample_configuration') + flexmock(module).should_receive('merge_source_configuration_into_destination') + flexmock(module).should_receive('render_configuration') + flexmock(module).should_receive('_comment_out_optional_configuration') + flexmock(module).should_receive('write_configuration').never() + + module.generate_sample_configuration(True, None, 'dest.yaml', 'schema.yaml') diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index 8c2063a5..642eaf6b 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -124,4 +124,5 @@ def test_run_bootstrap_does_not_raise(): flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( 'archive' ) + module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version) diff --git a/tests/unit/actions/config/test_generate.py b/tests/unit/actions/config/test_generate.py new file mode 100644 index 00000000..5b82dd35 --- /dev/null +++ b/tests/unit/actions/config/test_generate.py @@ -0,0 +1,39 @@ +from flexmock import flexmock + +from borgmatic.actions.config import generate as module + + +def test_run_bootstrap_does_not_raise(): + generate_arguments = flexmock( + source_filename=None, + destination_filename='destination.yaml', + overwrite=False, + ) + global_arguments = flexmock(dry_run=False) + flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration') + + module.run_generate(generate_arguments, global_arguments) + + +def test_run_bootstrap_with_dry_run_does_not_raise(): + generate_arguments = flexmock( + source_filename=None, + destination_filename='destination.yaml', + overwrite=False, + ) + global_arguments = flexmock(dry_run=True) + flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration') + + module.run_generate(generate_arguments, global_arguments) + + +def test_run_bootstrap_with_source_filename_does_not_raise(): + generate_arguments = flexmock( + source_filename='source.yaml', + destination_filename='destination.yaml', + overwrite=False, + ) + global_arguments = flexmock(dry_run=False) + flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration') + + module.run_generate(generate_arguments, global_arguments) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 5ac06326..f5d7afb4 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -962,6 +962,81 @@ def test_get_local_path_without_local_path_defaults_to_borg(): assert module.get_local_path({'test.yaml': {'location': {}}}) == 'borg' +def test_collect_highlander_action_summary_logs_info_for_success_with_bootstrap(): + flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) + flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap') + arguments = { + 'bootstrap': flexmock(repository='repo'), + 'global': flexmock(dry_run=False), + } + + logs = tuple( + module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + ) + assert {log.levelno for log in logs} == {logging.ANSWER} + + +def test_collect_highlander_action_summary_logs_error_on_bootstrap_failure(): + flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) + flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').and_raise( + ValueError + ) + arguments = { + 'bootstrap': flexmock(repository='repo'), + 'global': flexmock(dry_run=False), + } + + logs = tuple( + module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + + +def test_collect_highlander_action_summary_logs_error_on_bootstrap_local_borg_version_failure(): + flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError) + flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').never() + arguments = { + 'bootstrap': flexmock(repository='repo'), + 'global': flexmock(dry_run=False), + } + + logs = tuple( + module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + + +def test_collect_highlander_action_summary_logs_info_for_success_with_generate(): + flexmock(module.borgmatic.actions.config.generate).should_receive('run_generate') + arguments = { + 'generate': flexmock(destination='test.yaml'), + 'global': flexmock(dry_run=False), + } + + logs = tuple( + module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + ) + assert {log.levelno for log in logs} == {logging.ANSWER} + + +def test_collect_highlander_action_summary_logs_error_on_generate_failure(): + flexmock(module.borgmatic.actions.config.generate).should_receive('run_generate').and_raise( + ValueError + ) + arguments = { + 'generate': flexmock(destination='test.yaml'), + 'global': flexmock(dry_run=False), + } + + logs = tuple( + module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + + def test_collect_configuration_run_summary_logs_info_for_success(): flexmock(module.command).should_receive('execute_hook').never() flexmock(module.validate).should_receive('guard_configuration_contains_repository') @@ -1000,41 +1075,6 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): assert {log.levelno for log in logs} == {logging.INFO} -def test_collect_configuration_run_summary_logs_info_for_success_with_bootstrap(): - flexmock(module.validate).should_receive('guard_single_repository_selected').never() - flexmock(module.validate).should_receive('guard_configuration_contains_repository').never() - flexmock(module).should_receive('run_configuration').never() - flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap') - arguments = { - 'bootstrap': flexmock(repository='repo'), - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - } - - logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) - ) - assert {log.levelno for log in logs} == {logging.ANSWER} - - -def test_collect_configuration_run_summary_logs_error_on_bootstrap_failure(): - flexmock(module.validate).should_receive('guard_single_repository_selected').never() - flexmock(module.validate).should_receive('guard_configuration_contains_repository').never() - flexmock(module).should_receive('run_configuration').never() - flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').and_raise( - ValueError - ) - arguments = { - 'bootstrap': flexmock(repository='repo'), - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - } - - logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) - ) - - assert {log.levelno for log in logs} == {logging.CRITICAL} - - def test_collect_configuration_run_summary_logs_extract_with_repository_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise( ValueError From bc93401a70526db8a47e339a445e896426a58f8b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 21 Jun 2023 13:14:54 -0700 Subject: [PATCH 289/344] Codespell fixes. --- docs/how-to/make-per-application-backups.md | 4 ++-- docs/how-to/set-up-backups.md | 2 +- docs/how-to/upgrade.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 1f725491..1cb37ba0 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -33,8 +33,8 @@ sudo borgmatic config generate --destination /etc/borgmatic.d/repo2.yaml ``` Prior to version 1.7.15 The -command to generate configuation files was `generate-borgmatic-config` instead -of `borgmatic config generate`. +command to generate configuration files was `generate-borgmatic-config` +instead of `borgmatic config generate`. When you set up multiple configuration files like this, borgmatic will run each one in turn from a single borgmatic invocation. This includes, by diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index dcae31c0..f178400e 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -125,7 +125,7 @@ sudo borgmatic config generate ``` Prior to version 1.7.15 -Generate a configuation file with this command instead: +Generate a configuration file with this command instead: ```bash sudo generate-borgmatic-config diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index 3cd12601..be85880f 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -41,8 +41,8 @@ borgmatic config generate --source config.yaml --destination config-new.yaml ``` Prior to version 1.7.15 The -command to generate configuation files was `generate-borgmatic-config` instead -of `borgmatic config generate`. +command to generate configuration files was `generate-borgmatic-config` +instead of `borgmatic config generate`. New options start as commented out, so you can edit the file and decide whether you want to use each one. From 01fffab898adb4bf77508aa5418d020e45388781 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 22 Jun 2023 09:12:06 -0700 Subject: [PATCH 290/344] Clarify that references docs are only for the most recent version of borgmatic. --- docs/reference/command-line.md | 12 +++++++++--- docs/reference/configuration.md | 10 ++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/reference/command-line.md b/docs/reference/command-line.md index 9cfdd7ee..08ccd601 100644 --- a/docs/reference/command-line.md +++ b/docs/reference/command-line.md @@ -7,11 +7,17 @@ eleventyNavigation: --- ## borgmatic options -Here are all of the available borgmatic command-line options, including the -separate options for each action sub-command. Note that most of the -flags listed here do not have equivalents in borgmatic's [configuration +Here are all of the available borgmatic command-line flags for the most recent +version of borgmatic, including the separate flags for each action +(sub-command). Most of the flags listed here do not have equivalents in +borgmatic's [configuration file](https://torsion.org/borgmatic/docs/reference/configuration/). +If you're using an older version of borgmatic, some of these flags may not be +present in that version, and you should instead use `borgmatic --help` or +`borgmatic [action name] --help` (where `[action name]` is the name of an +action like `list`, `create`, etc.). + ``` {% include borgmatic/command-line.txt %} ``` diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 316e58f0..d82eb8ed 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -7,11 +7,17 @@ eleventyNavigation: --- ## Configuration file -Here is a full sample borgmatic configuration file including all available options: +Below is a sample borgmatic configuration file including all available options +for the most recent version of borgmatic. This file is also [available for +download](https://torsion.org/borgmatic/docs/reference/config.yaml). + +If you're an older version of borgmatic, some of these options may not work, +and you should instead [generate a sample configuration file specific to your +borgmatic +version](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#configuration). ```yaml {% include borgmatic/config.yaml %} ``` Note that you can also [download this configuration -file](https://torsion.org/borgmatic/docs/reference/config.yaml) for use locally. From 3addb60fb8c1f047180fa94a62b59b417645fa71 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 22 Jun 2023 09:13:45 -0700 Subject: [PATCH 291/344] Actually link to the most recent version. --- docs/reference/command-line.md | 9 +++++---- docs/reference/configuration.md | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/reference/command-line.md b/docs/reference/command-line.md index 08ccd601..5ef3836c 100644 --- a/docs/reference/command-line.md +++ b/docs/reference/command-line.md @@ -7,10 +7,11 @@ eleventyNavigation: --- ## borgmatic options -Here are all of the available borgmatic command-line flags for the most recent -version of borgmatic, including the separate flags for each action -(sub-command). Most of the flags listed here do not have equivalents in -borgmatic's [configuration +Here are all of the available borgmatic command-line flags for the [most +recent version of +borgmatic](https://projects.torsion.org/borgmatic-collective/borgmatic/releases), +including the separate flags for each action (sub-command). Most of the flags +listed here do not have equivalents in borgmatic's [configuration file](https://torsion.org/borgmatic/docs/reference/configuration/). If you're using an older version of borgmatic, some of these flags may not be diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index d82eb8ed..1365898b 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -8,7 +8,9 @@ eleventyNavigation: ## Configuration file Below is a sample borgmatic configuration file including all available options -for the most recent version of borgmatic. This file is also [available for +for the [most recent version of +borgmatic](https://projects.torsion.org/borgmatic-collective/borgmatic/releases). +This file is also [available for download](https://torsion.org/borgmatic/docs/reference/config.yaml). If you're an older version of borgmatic, some of these options may not work, From 248500c7bec9195492a0ff2840d192564da23867 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 22 Jun 2023 09:21:06 -0700 Subject: [PATCH 292/344] Accidentally a word. --- docs/reference/configuration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1365898b..262501d9 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -13,9 +13,9 @@ borgmatic](https://projects.torsion.org/borgmatic-collective/borgmatic/releases) This file is also [available for download](https://torsion.org/borgmatic/docs/reference/config.yaml). -If you're an older version of borgmatic, some of these options may not work, -and you should instead [generate a sample configuration file specific to your -borgmatic +If you're using an older version of borgmatic, some of these options may not +work, and you should instead [generate a sample configuration file specific to +your borgmatic version](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#configuration). ```yaml From 69611681e2f912541694ebaf7fa3a7267ab43c75 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 22 Jun 2023 12:40:57 -0700 Subject: [PATCH 293/344] Add database restore overrides to NEWS, add a test, and move some tests (#326). --- NEWS | 2 + tests/unit/hooks/test_postgresql.py | 77 ++++++++++++++++------------- tests/unit/hooks/test_sqlite.py | 1 + 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/NEWS b/NEWS index b6939ffa..7e78c131 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.7.15.dev0 + * #326: Add configuration options and command-line flags for backing up a database from one + location while restoring it somewhere else. * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors. * #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style configuration. diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index e48258ee..7ba45847 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -6,6 +6,50 @@ from flexmock import flexmock from borgmatic.hooks import postgresql as module +def test_make_extra_environment_maps_options_to_environment(): + database = { + 'name': 'foo', + 'password': 'pass', + 'ssl_mode': 'require', + 'ssl_cert': 'cert.crt', + 'ssl_key': 'key.key', + 'ssl_root_cert': 'root.crt', + 'ssl_crl': 'crl.crl', + } + expected = { + 'PGPASSWORD': 'pass', + 'PGSSLMODE': 'require', + 'PGSSLCERT': 'cert.crt', + 'PGSSLKEY': 'key.key', + 'PGSSLROOTCERT': 'root.crt', + 'PGSSLCRL': 'crl.crl', + } + + extra_env = module.make_extra_environment(database) + + assert extra_env == expected + + +def test_make_extra_environment_with_cli_password_sets_correct_password(): + database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'} + + extra = module.make_extra_environment( + database, restore_connection_params={'password': 'clipassword'} + ) + + assert extra['PGPASSWORD'] == 'clipassword' + + +def test_make_extra_environment_without_cli_password_or_configured_password_does_not_set_password(): + database = {'name': 'foo'} + + extra = module.make_extra_environment( + database, restore_connection_params={'username': 'someone'} + ) + + assert 'PGPASSWORD' not in extra + + def test_database_names_to_dump_passes_through_individual_database_name(): database = {'name': 'foo'} @@ -301,29 +345,6 @@ def test_dump_databases_runs_pg_dump_with_username_and_password(): assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] -def test_make_extra_environment_maps_options_to_environment(): - database = { - 'name': 'foo', - 'password': 'pass', - 'ssl_mode': 'require', - 'ssl_cert': 'cert.crt', - 'ssl_key': 'key.key', - 'ssl_root_cert': 'root.crt', - 'ssl_crl': 'crl.crl', - } - expected = { - 'PGPASSWORD': 'pass', - 'PGSSLMODE': 'require', - 'PGSSLCERT': 'cert.crt', - 'PGSSLKEY': 'key.key', - 'PGSSLROOTCERT': 'root.crt', - 'PGSSLCRL': 'crl.crl', - } - - extra_env = module.make_extra_environment(database) - assert extra_env == expected - - def test_dump_databases_runs_pg_dump_with_directory_format(): databases = [{'name': 'foo', 'format': 'directory'}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -638,16 +659,6 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): ) -def test_make_extra_environment_with_cli_password_sets_correct_password(): - database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'} - - extra = module.make_extra_environment( - database, restore_connection_params={'password': 'clipassword'} - ) - - assert extra['PGPASSWORD'] == 'clipassword' - - def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): database_config = [ { diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py index 5820713b..33317372 100644 --- a/tests/unit/hooks/test_sqlite.py +++ b/tests/unit/hooks/test_sqlite.py @@ -1,4 +1,5 @@ import logging + import pytest from flexmock import flexmock From e8c862659c4cd4c5d73a7d6cf9ba45a8e2bc024d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 22 Jun 2023 14:20:42 -0700 Subject: [PATCH 294/344] Add missing services to build service configuration and add a test to catch this in the future (#326). --- .drone.yml | 22 +++++++++++++++++++ .../test_dev_parity_with_build_server.py | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/end-to-end/test_dev_parity_with_build_server.py diff --git a/.drone.yml b/.drone.yml index dcc19c67..de345880 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,16 +8,38 @@ services: environment: POSTGRES_PASSWORD: test POSTGRES_DB: test + - name: postgresql2 + image: docker.io/postgres:13.1-alpine + environment: + POSTGRES_PASSWORD: test2 + POSTGRES_DB: test + POSTGRES_USER: postgres2 + commands: + - -p 5433 - name: mysql image: docker.io/mariadb:10.5 environment: MYSQL_ROOT_PASSWORD: test MYSQL_DATABASE: test + - name: mysql2 + image: docker.io/mariadb:10.5 + environment: + MYSQL_ROOT_PASSWORD: test2 + MYSQL_DATABASE: test + commands: + - --port=3307 - name: mongodb image: docker.io/mongo:5.0.5 environment: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: test + - name: mongodb2 + image: docker.io/mongo:5.0.5 + environment: + MONGO_INITDB_ROOT_USERNAME: root2 + MONGO_INITDB_ROOT_PASSWORD: test2 + commands: + - --port=27018 clone: skip_verify: true diff --git a/tests/end-to-end/test_dev_parity_with_build_server.py b/tests/end-to-end/test_dev_parity_with_build_server.py new file mode 100644 index 00000000..b555490b --- /dev/null +++ b/tests/end-to-end/test_dev_parity_with_build_server.py @@ -0,0 +1,22 @@ +import ruamel.yaml + + +def test_dev_docker_compose_has_same_services_as_build_server_configuration(): + yaml = ruamel.yaml.YAML(typ='safe') + dev_services = { + name: service + for name, service in yaml.load(open('tests/end-to-end/docker-compose.yaml').read())['services'].items() + if name != 'tests' + } + build_server_services = tuple(yaml.load_all(open('.drone.yml').read()))[0]['services'] + + assert len(dev_services) == len(build_server_services) + + for build_service in build_server_services: + dev_service = dev_services[build_service['name']] + assert dev_service['image'] == build_service['image'] + assert dev_service['environment'] == build_service['environment'] + + if 'command' in dev_service or 'commands' in build_service: + assert len(build_service['commands']) <= 1 + assert dev_service['command'] == build_service['commands'][0] From 62a2f5a1d0c8afea95636aef53d57435734ec60a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 22 Jun 2023 14:25:26 -0700 Subject: [PATCH 295/344] Code formatting. --- tests/end-to-end/test_dev_parity_with_build_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/end-to-end/test_dev_parity_with_build_server.py b/tests/end-to-end/test_dev_parity_with_build_server.py index b555490b..f8bd00ff 100644 --- a/tests/end-to-end/test_dev_parity_with_build_server.py +++ b/tests/end-to-end/test_dev_parity_with_build_server.py @@ -5,7 +5,9 @@ def test_dev_docker_compose_has_same_services_as_build_server_configuration(): yaml = ruamel.yaml.YAML(typ='safe') dev_services = { name: service - for name, service in yaml.load(open('tests/end-to-end/docker-compose.yaml').read())['services'].items() + for name, service in yaml.load(open('tests/end-to-end/docker-compose.yaml').read())[ + 'services' + ].items() if name != 'tests' } build_server_services = tuple(yaml.load_all(open('.drone.yml').read()))[0]['services'] From 308c96aeb58530d37143558a615b783cb293782b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 22 Jun 2023 14:37:08 -0700 Subject: [PATCH 296/344] Add comment describing need for dev-CI parity test. --- .../test_dev_parity_with_build_server.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/end-to-end/test_dev_parity_with_build_server.py b/tests/end-to-end/test_dev_parity_with_build_server.py index f8bd00ff..aa01ff0f 100644 --- a/tests/end-to-end/test_dev_parity_with_build_server.py +++ b/tests/end-to-end/test_dev_parity_with_build_server.py @@ -2,6 +2,20 @@ import ruamel.yaml def test_dev_docker_compose_has_same_services_as_build_server_configuration(): + ''' + The end-to-end test configuration for local development and the build server's test + configuration use two different mechanisms for configuring and spinning up "services"—the + database containers upon which the end-to-end tests are reliant. The dev configuration uses + Docker Compose, while the Drone build server configuration uses its own similar-but-different + configuration file format. + + Therefore, to ensure dev-build parity, these tests assert that the services are the same across + the dev and build configurations. This includes service name, container image, environment + variables, and commands. + + This test only compares services and does not assert anything else about the respective testing + environments. + ''' yaml = ruamel.yaml.YAML(typ='safe') dev_services = { name: service From bb0dd14f69b735935bfd29f0b968f1093489aee4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 22 Jun 2023 14:55:32 -0700 Subject: [PATCH 297/344] Attempt to fix CI test failures (#326). --- .drone.yml | 6 +++--- tests/end-to-end/docker-compose.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index de345880..519f2bf6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,7 +15,7 @@ services: POSTGRES_DB: test POSTGRES_USER: postgres2 commands: - - -p 5433 + - postgres -p 5433 - name: mysql image: docker.io/mariadb:10.5 environment: @@ -27,7 +27,7 @@ services: MYSQL_ROOT_PASSWORD: test2 MYSQL_DATABASE: test commands: - - --port=3307 + - mysqld --port=3307 - name: mongodb image: docker.io/mongo:5.0.5 environment: @@ -39,7 +39,7 @@ services: MONGO_INITDB_ROOT_USERNAME: root2 MONGO_INITDB_ROOT_PASSWORD: test2 commands: - - --port=27018 + - mongod --port=27018 clone: skip_verify: true diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index bbeb29f2..a0d9bff7 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -11,7 +11,7 @@ services: POSTGRES_PASSWORD: test2 POSTGRES_DB: test POSTGRES_USER: postgres2 - command: -p 5433 + command: postgres -p 5433 mysql: image: docker.io/mariadb:10.5 environment: @@ -22,7 +22,7 @@ services: environment: MYSQL_ROOT_PASSWORD: test2 MYSQL_DATABASE: test - command: --port=3307 + command: mysqld --port=3307 mongodb: image: docker.io/mongo:5.0.5 environment: @@ -33,7 +33,7 @@ services: environment: MONGO_INITDB_ROOT_USERNAME: root2 MONGO_INITDB_ROOT_PASSWORD: test2 - command: --port=27018 + command: mongod --port=27018 tests: image: docker.io/alpine:3.13 environment: From 23809e90605c476ef042e593d9c3906ce16000f1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 22 Jun 2023 15:11:49 -0700 Subject: [PATCH 298/344] More Docker build fun (#326). --- .drone.yml | 6 +++--- tests/end-to-end/docker-compose.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index 519f2bf6..e353b3c7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,7 +15,7 @@ services: POSTGRES_DB: test POSTGRES_USER: postgres2 commands: - - postgres -p 5433 + - docker-entrypoint.sh -p 5433 - name: mysql image: docker.io/mariadb:10.5 environment: @@ -27,7 +27,7 @@ services: MYSQL_ROOT_PASSWORD: test2 MYSQL_DATABASE: test commands: - - mysqld --port=3307 + - docker-entrypoint.sh --port=3307 - name: mongodb image: docker.io/mongo:5.0.5 environment: @@ -39,7 +39,7 @@ services: MONGO_INITDB_ROOT_USERNAME: root2 MONGO_INITDB_ROOT_PASSWORD: test2 commands: - - mongod --port=27018 + - docker-entrypoint.sh --port=27018 clone: skip_verify: true diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index a0d9bff7..8753ddd6 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -11,7 +11,7 @@ services: POSTGRES_PASSWORD: test2 POSTGRES_DB: test POSTGRES_USER: postgres2 - command: postgres -p 5433 + command: docker-entrypoint.sh -p 5433 mysql: image: docker.io/mariadb:10.5 environment: @@ -22,7 +22,7 @@ services: environment: MYSQL_ROOT_PASSWORD: test2 MYSQL_DATABASE: test - command: mysqld --port=3307 + command: docker-entrypoint.sh --port=3307 mongodb: image: docker.io/mongo:5.0.5 environment: @@ -33,7 +33,7 @@ services: environment: MONGO_INITDB_ROOT_USERNAME: root2 MONGO_INITDB_ROOT_PASSWORD: test2 - command: mongod --port=27018 + command: docker-entrypoint.sh --port=27018 tests: image: docker.io/alpine:3.13 environment: From e4e455ee457f2429b52287144618e1879bc3b234 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 23 Jun 2023 10:11:41 -0700 Subject: [PATCH 299/344] Deprecate validate-borgmatic-config in favor of new "config validate" action (#529). --- NEWS | 3 +- borgmatic/actions/config/generate.py | 7 ++ borgmatic/actions/config/validate.py | 22 ++++++ borgmatic/commands/arguments.py | 21 +++++- borgmatic/commands/borgmatic.py | 72 +++++++++++++++--- borgmatic/commands/validate_config.py | 71 +++--------------- docs/Dockerfile | 2 +- docs/how-to/make-per-application-backups.md | 11 ++- docs/how-to/set-up-backups.md | 7 ++ tests/integration/actions/__init__.py | 0 tests/integration/actions/config/__init__.py | 0 .../actions/config/test_validate.py | 37 +++++++++ .../commands/test_validate_config.py | 26 +------ tests/unit/actions/config/test_generate.py | 6 +- tests/unit/actions/config/test_validate.py | 17 +++++ tests/unit/commands/test_borgmatic.py | 75 +++++++++++++++++-- 16 files changed, 266 insertions(+), 111 deletions(-) create mode 100644 borgmatic/actions/config/validate.py create mode 100644 tests/integration/actions/__init__.py create mode 100644 tests/integration/actions/config/__init__.py create mode 100644 tests/integration/actions/config/test_validate.py create mode 100644 tests/unit/actions/config/test_validate.py diff --git a/NEWS b/NEWS index 7e78c131..781e30dd 100644 --- a/NEWS +++ b/NEWS @@ -4,7 +4,8 @@ * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors. * #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style configuration. - * #529: Deprecate generate-borgmatic-config in favor if new "config generate" action. + * #529: Deprecate generate-borgmatic-config in favor of new "config generate" action. + * #529: Deprecate validate-borgmatic-config in favor of new "config validate" action. * #697, #712: Extract borgmatic configuration from backup via new "config bootstrap" action—even when borgmatic has no configuration yet! * #669: Add sample systemd user service for running borgmatic as a non-root user. diff --git a/borgmatic/actions/config/generate.py b/borgmatic/actions/config/generate.py index 5f430383..1943ea74 100644 --- a/borgmatic/actions/config/generate.py +++ b/borgmatic/actions/config/generate.py @@ -7,6 +7,13 @@ logger = logging.getLogger(__name__) def run_generate(generate_arguments, global_arguments): + ''' + Given the generate arguments and the global arguments, each as an argparse.Namespace instance, + run the "generate" action. + + Raise FileExistsError if a file already exists at the destination path and the generate + arguments do not have overwrite set. + ''' dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else '' logger.answer( diff --git a/borgmatic/actions/config/validate.py b/borgmatic/actions/config/validate.py new file mode 100644 index 00000000..2cec6135 --- /dev/null +++ b/borgmatic/actions/config/validate.py @@ -0,0 +1,22 @@ +import logging + +import borgmatic.config.generate + +logger = logging.getLogger(__name__) + + +def run_validate(validate_arguments, configs): + ''' + Given the validate arguments as an argparse.Namespace instance and a dict of configuration + filename to corresponding parsed configuration, run the "validate" action. + + Most of the validation is actually performed implicitly by the standard borgmatic configuration + loading machinery prior to here, so this function mainly exists to support additional validate + flags like "--show". + ''' + if validate_arguments.show: + for config_path, config in configs.items(): + if len(configs) > 1: + logger.answer('---') + + logger.answer(borgmatic.config.generate.render_configuration(config)) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index ae6056a2..fe8c1dd1 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -699,8 +699,8 @@ def make_parsers(): config_bootstrap_parser = config_parsers.add_parser( 'bootstrap', - help='Extract the borgmatic config files from a named archive', - description='Extract the borgmatic config files from a named archive', + help='Extract the borgmatic configuration files from a named archive', + description='Extract the borgmatic configuration files from a named archive', add_help=False, ) config_bootstrap_group = config_bootstrap_parser.add_argument_group( @@ -774,6 +774,23 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + config_validate_parser = config_parsers.add_parser( + 'validate', + help='Validate that borgmatic configuration files specified with --config (see borgmatic --help)', + description='Validate borgmatic configuration files specified with --config (see borgmatic --help)', + add_help=False, + ) + config_validate_group = config_validate_parser.add_argument_group('config validate arguments') + config_validate_group.add_argument( + '-s', + '--show', + action='store_true', + help='Show the validated configuration after all include merging has occurred', + ) + config_validate_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) + export_tar_parser = action_parsers.add_parser( 'export-tar', aliases=ACTION_ALIASES['export-tar'], diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index e04a785a..7d31ddee 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -20,6 +20,7 @@ import borgmatic.actions.check import borgmatic.actions.compact import borgmatic.actions.config.bootstrap import borgmatic.actions.config.generate +import borgmatic.actions.config.validate import borgmatic.actions.create import borgmatic.actions.export_tar import borgmatic.actions.extract @@ -500,6 +501,9 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): Given a sequence of configuration filenames, load and validate each configuration file. Return the results as a tuple of: dict of configuration filename to corresponding parsed configuration, and sequence of logging.LogRecord instances containing any parse errors. + + Log records are returned here instead of being logged directly because logging isn't yet + initialized at this point! ''' # Dict mapping from config filename to corresponding parsed config dict. configs = collections.OrderedDict() @@ -507,6 +511,17 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): # Parse and load each configuration file. for config_filename in config_filenames: + logs.extend( + [ + logging.makeLogRecord( + dict( + levelno=logging.DEBUG, + levelname='DEBUG', + msg=f'{config_filename}: Loading configuration file', + ) + ), + ] + ) try: configs[config_filename], parse_logs = validate.parse_configuration( config_filename, validate.schema_filename(), overrides, resolve_env @@ -603,12 +618,13 @@ def get_local_path(configs): return next(iter(configs.values())).get('location', {}).get('local_path', 'borg') -def collect_highlander_action_summary_logs(configs, arguments): +def collect_highlander_action_summary_logs(configs, arguments, configuration_parse_errors): ''' - Given a dict of configuration filename to corresponding parsed configuration and parsed - command-line arguments as a dict from subparser name to a parsed namespace of arguments, run - a highlander action specified in the arguments, if any, and yield a series of logging.LogRecord - instances containing summary information. + Given a dict of configuration filename to corresponding parsed configuration, parsed + command-line arguments as a dict from subparser name to a parsed namespace of arguments, and + whether any configuration files encountered errors during parsing, run a highlander action + specified in the arguments, if any, and yield a series of logging.LogRecord instances containing + summary information. A highlander action is an action that cannot coexist with other actions on the borgmatic command-line, and borgmatic exits after processing such an action. @@ -662,6 +678,37 @@ def collect_highlander_action_summary_logs(configs, arguments): return + if 'validate' in arguments: + if configuration_parse_errors: + yield logging.makeLogRecord( + dict( + levelno=logging.CRITICAL, + levelname='CRITICAL', + msg='Configuration validation failed', + ) + ) + + return + + try: + borgmatic.actions.config.validate.run_validate(arguments['validate'], configs) + + yield logging.makeLogRecord( + dict( + levelno=logging.ANSWER, + levelname='ANSWER', + msg='All configuration files are valid', + ) + ) + except ( + CalledProcessError, + ValueError, + OSError, + ) as error: + yield from log_error_records(error) + + return + def collect_configuration_run_summary_logs(configs, arguments): ''' @@ -800,6 +847,9 @@ def main(extra_summary_logs=[]): # pragma: no cover configs, parse_logs = load_configurations( config_filenames, global_arguments.overrides, global_arguments.resolve_env ) + configuration_parse_errors = ( + (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False + ) any_json_flags = any( getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values() @@ -822,15 +872,17 @@ def main(extra_summary_logs=[]): # pragma: no cover logger.critical(f'Error configuring logging: {error}') exit_with_help_link() - logger.debug('Ensuring legacy configuration is upgraded') - summary_logs = ( - parse_logs + extra_summary_logs + + parse_logs + ( - list(collect_highlander_action_summary_logs(configs, arguments)) + list( + collect_highlander_action_summary_logs( + configs, arguments, configuration_parse_errors + ) + ) or list(collect_configuration_run_summary_logs(configs, arguments)) ) - + extra_summary_logs ) summary_logs_max_level = max(log.levelno for log in summary_logs) diff --git a/borgmatic/commands/validate_config.py b/borgmatic/commands/validate_config.py index 8aa8d321..0b3dd1ff 100644 --- a/borgmatic/commands/validate_config.py +++ b/borgmatic/commands/validate_config.py @@ -1,68 +1,17 @@ import logging import sys -from argparse import ArgumentParser -import borgmatic.config.generate -from borgmatic.config import collect, validate - -logger = logging.getLogger(__name__) +import borgmatic.commands.borgmatic -def parse_arguments(*arguments): - ''' - Given command-line arguments with which this script was invoked, parse the arguments and return - them as an ArgumentParser instance. - ''' - config_paths = collect.get_default_config_paths() - - parser = ArgumentParser(description='Validate borgmatic configuration file(s).') - parser.add_argument( - '-c', - '--config', - nargs='+', - dest='config_paths', - default=config_paths, - help=f'Configuration filenames or directories, defaults to: {config_paths}', - ) - parser.add_argument( - '-s', - '--show', - action='store_true', - help='Show the validated configuration after all include merging has occurred', +def main(): + warning_log = logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg='validate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config validate" instead.', + ) ) - return parser.parse_args(arguments) - - -def main(): # pragma: no cover - arguments = parse_arguments(*sys.argv[1:]) - - logging.basicConfig(level=logging.INFO, format='%(message)s') - - config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths)) - if len(config_filenames) == 0: - logger.critical('No files to validate found') - sys.exit(1) - - found_issues = False - for config_filename in config_filenames: - try: - config, parse_logs = validate.parse_configuration( - config_filename, validate.schema_filename() - ) - except (ValueError, OSError, validate.Validation_error) as error: - logging.critical(f'{config_filename}: Error parsing configuration file') - logging.critical(error) - found_issues = True - else: - for log in parse_logs: - logger.handle(log) - - if arguments.show: - print('---') - print(borgmatic.config.generate.render_configuration(config)) - - if found_issues: - sys.exit(1) - - logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}") + sys.argv = ['borgmatic', 'config', 'validate'] + sys.argv[1:] + borgmatic.commands.borgmatic.main([warning_log]) diff --git a/docs/Dockerfile b/docs/Dockerfile index 4ac1867b..118768ce 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -4,7 +4,7 @@ COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ - && for action in rcreate transfer create prune compact check extract config "config bootstrap" "config generate" export-tar mount umount restore rlist list rinfo info break-lock borg; do \ + && for action in rcreate transfer create prune compact check extract config "config bootstrap" "config generate" "config validate" export-tar mount umount restore rlist list rinfo info break-lock borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic $action --help >> /command-line.txt; done diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 1cb37ba0..fb815d87 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -399,9 +399,16 @@ includes. ## Debugging includes -New in version 1.7.12 If you'd +New in version 1.7.15 If you'd like to see what the loaded configuration looks like after includes get merged -in, run `validate-borgmatic-config` on your configuration file: +in, run the `validate` action on your configuration file: + +```bash +sudo borgmatic config validate --show +``` + +In version 1.7.12 through +1.7.14 Use this command instead: ```bash sudo validate-borgmatic-config --show diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index f178400e..7fe69fd9 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -185,6 +185,13 @@ redundant](https://torsion.org/borgmatic/docs/how-to/make-backups-redundant/). If you'd like to validate that your borgmatic configuration is valid, the following command is available for that: +```bash +sudo borgmatic config validate +``` + +Prior to version 1.7.15 +Validate a configuration file with this command instead: + ```bash sudo validate-borgmatic-config ``` diff --git a/tests/integration/actions/__init__.py b/tests/integration/actions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/actions/config/__init__.py b/tests/integration/actions/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/actions/config/test_validate.py b/tests/integration/actions/config/test_validate.py new file mode 100644 index 00000000..4dbfd2a3 --- /dev/null +++ b/tests/integration/actions/config/test_validate.py @@ -0,0 +1,37 @@ +import argparse + +from flexmock import flexmock + +import borgmatic.logger +from borgmatic.actions.config import validate as module + + +def test_run_validate_with_show_renders_configurations(): + log_lines = [] + borgmatic.logger.add_custom_log_levels() + + def fake_logger_answer(message): + log_lines.append(message) + + flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer) + + module.run_validate(argparse.Namespace(show=True), {'test.yaml': {'foo': {'bar': 'baz'}}}) + + assert log_lines == ['''foo:\n bar: baz\n'''] + + +def test_run_validate_with_show_and_multiple_configs_renders_each(): + log_lines = [] + borgmatic.logger.add_custom_log_levels() + + def fake_logger_answer(message): + log_lines.append(message) + + flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer) + + module.run_validate( + argparse.Namespace(show=True), + {'test.yaml': {'foo': {'bar': 'baz'}}, 'other.yaml': {'quux': 'value'}}, + ) + + assert log_lines == ['---', 'foo:\n bar: baz\n', '---', 'quux: value\n'] diff --git a/tests/integration/commands/test_validate_config.py b/tests/integration/commands/test_validate_config.py index 78887e71..1acd332a 100644 --- a/tests/integration/commands/test_validate_config.py +++ b/tests/integration/commands/test_validate_config.py @@ -3,27 +3,7 @@ from flexmock import flexmock from borgmatic.commands import validate_config as module -def test_parse_arguments_with_no_arguments_uses_defaults(): - config_paths = ['default'] - flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) +def test_main_does_not_raise(): + flexmock(module.borgmatic.commands.borgmatic).should_receive('main') - parser = module.parse_arguments() - - assert parser.config_paths == config_paths - - -def test_parse_arguments_with_multiple_config_paths_parses_as_list(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - parser = module.parse_arguments('--config', 'myconfig', 'otherconfig') - - assert parser.config_paths == ['myconfig', 'otherconfig'] - - -def test_parse_arguments_supports_show_flag(): - config_paths = ['default'] - flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - - parser = module.parse_arguments('--config', 'myconfig', '--show') - - assert parser.show + module.main() diff --git a/tests/unit/actions/config/test_generate.py b/tests/unit/actions/config/test_generate.py index 5b82dd35..d25a58e9 100644 --- a/tests/unit/actions/config/test_generate.py +++ b/tests/unit/actions/config/test_generate.py @@ -3,7 +3,7 @@ from flexmock import flexmock from borgmatic.actions.config import generate as module -def test_run_bootstrap_does_not_raise(): +def test_run_generate_does_not_raise(): generate_arguments = flexmock( source_filename=None, destination_filename='destination.yaml', @@ -15,7 +15,7 @@ def test_run_bootstrap_does_not_raise(): module.run_generate(generate_arguments, global_arguments) -def test_run_bootstrap_with_dry_run_does_not_raise(): +def test_run_generate_with_dry_run_does_not_raise(): generate_arguments = flexmock( source_filename=None, destination_filename='destination.yaml', @@ -27,7 +27,7 @@ def test_run_bootstrap_with_dry_run_does_not_raise(): module.run_generate(generate_arguments, global_arguments) -def test_run_bootstrap_with_source_filename_does_not_raise(): +def test_run_generate_with_source_filename_does_not_raise(): generate_arguments = flexmock( source_filename='source.yaml', destination_filename='destination.yaml', diff --git a/tests/unit/actions/config/test_validate.py b/tests/unit/actions/config/test_validate.py new file mode 100644 index 00000000..862d1bfa --- /dev/null +++ b/tests/unit/actions/config/test_validate.py @@ -0,0 +1,17 @@ +from flexmock import flexmock + +from borgmatic.actions.config import validate as module + + +def test_run_validate_does_not_raise(): + validate_arguments = flexmock(show=False) + flexmock(module.borgmatic.config.generate).should_receive('render_configuration') + + module.run_validate(validate_arguments, flexmock()) + + +def test_run_validate_with_show_does_not_raise(): + validate_arguments = flexmock(show=True) + flexmock(module.borgmatic.config.generate).should_receive('render_configuration') + + module.run_validate(validate_arguments, {'test.yaml': flexmock(), 'other.yaml': flexmock()}) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index f5d7afb4..89eda4ce 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -877,7 +877,7 @@ def test_load_configurations_collects_parsed_configurations_and_logs(): configs, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml'))) assert configs == {'test.yaml': configuration, 'other.yaml': other_configuration} - assert logs == test_expected_logs + other_expected_logs + assert set(logs) >= set(test_expected_logs + other_expected_logs) def test_load_configurations_logs_warning_for_permission_error(): @@ -886,7 +886,7 @@ def test_load_configurations_logs_warning_for_permission_error(): configs, logs = tuple(module.load_configurations(('test.yaml',))) assert configs == {} - assert {log.levelno for log in logs} == {logging.WARNING} + assert max(log.levelno for log in logs) == logging.WARNING def test_load_configurations_logs_critical_for_parse_error(): @@ -895,7 +895,7 @@ def test_load_configurations_logs_critical_for_parse_error(): configs, logs = tuple(module.load_configurations(('test.yaml',))) assert configs == {} - assert {log.levelno for log in logs} == {logging.CRITICAL} + assert max(log.levelno for log in logs) == logging.CRITICAL def test_log_record_does_not_raise(): @@ -971,7 +971,9 @@ def test_collect_highlander_action_summary_logs_info_for_success_with_bootstrap( } logs = tuple( - module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False + ) ) assert {log.levelno for log in logs} == {logging.ANSWER} @@ -987,7 +989,9 @@ def test_collect_highlander_action_summary_logs_error_on_bootstrap_failure(): } logs = tuple( - module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False + ) ) assert {log.levelno for log in logs} == {logging.CRITICAL} @@ -1002,7 +1006,9 @@ def test_collect_highlander_action_summary_logs_error_on_bootstrap_local_borg_ve } logs = tuple( - module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False + ) ) assert {log.levelno for log in logs} == {logging.CRITICAL} @@ -1016,7 +1022,9 @@ def test_collect_highlander_action_summary_logs_info_for_success_with_generate() } logs = tuple( - module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False + ) ) assert {log.levelno for log in logs} == {logging.ANSWER} @@ -1031,7 +1039,58 @@ def test_collect_highlander_action_summary_logs_error_on_generate_failure(): } logs = tuple( - module.collect_highlander_action_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False + ) + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + + +def test_collect_highlander_action_summary_logs_info_for_success_with_validate(): + flexmock(module.borgmatic.actions.config.validate).should_receive('run_validate') + arguments = { + 'validate': flexmock(), + 'global': flexmock(dry_run=False), + } + + logs = tuple( + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False + ) + ) + assert {log.levelno for log in logs} == {logging.ANSWER} + + +def test_collect_highlander_action_summary_logs_error_on_validate_parse_failure(): + flexmock(module.borgmatic.actions.config.validate).should_receive('run_validate') + arguments = { + 'validate': flexmock(), + 'global': flexmock(dry_run=False), + } + + logs = tuple( + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=True + ) + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + + +def test_collect_highlander_action_summary_logs_error_on_run_validate_failure(): + flexmock(module.borgmatic.actions.config.validate).should_receive('run_validate').and_raise( + ValueError + ) + arguments = { + 'validate': flexmock(), + 'global': flexmock(dry_run=False), + } + + logs = tuple( + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False + ) ) assert {log.levelno for log in logs} == {logging.CRITICAL} From 35a11559acd39c84be84a7df605eff6dd61760b5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 24 Jun 2023 14:10:47 -0700 Subject: [PATCH 300/344] Fix error parsing arguments with multiple verbosity flags (#716). --- NEWS | 4 +- borgmatic/commands/arguments.py | 69 ++-- borgmatic/commands/completion/bash.py | 16 +- borgmatic/commands/completion/fish.py | 30 +- .../commands/completion/test_actions.py | 16 +- tests/integration/commands/test_arguments.py | 33 +- tests/unit/commands/test_arguments.py | 329 +++++++++++++++--- 7 files changed, 389 insertions(+), 108 deletions(-) diff --git a/NEWS b/NEWS index 781e30dd..f9d27518 100644 --- a/NEWS +++ b/NEWS @@ -6,8 +6,8 @@ configuration. * #529: Deprecate generate-borgmatic-config in favor of new "config generate" action. * #529: Deprecate validate-borgmatic-config in favor of new "config validate" action. - * #697, #712: Extract borgmatic configuration from backup via new "config bootstrap" action—even - when borgmatic has no configuration yet! + * #697, #712, #716: Extract borgmatic configuration from backup via new "config bootstrap" + action—even when borgmatic has no configuration yet! * #669: Add sample systemd user service for running borgmatic as a non-root user. * #711, #713: Fix an error when "data" check time files are accessed without getting upgraded first. diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index fe8c1dd1..eac88c8a 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -114,8 +114,8 @@ def parse_and_record_action_arguments( def get_unparsable_arguments(remaining_action_arguments): ''' - Given a sequence of argument tuples (one tuple per action parser that parsed arguments), - determine the remaining arguments that no action parsers have consumed. + Given a sequence of argument tuples (one per action parser that parsed arguments), determine the + remaining arguments that no action parsers have consumed. ''' if not remaining_action_arguments: return () @@ -129,14 +129,16 @@ def get_unparsable_arguments(remaining_action_arguments): ) -def parse_arguments_for_actions(unparsed_arguments, action_parsers): +def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parser): ''' - Given a sequence of arguments and a dict from action name to argparse.ArgumentParser - instance, give each requested action's parser a shot at parsing all arguments. This allows - common arguments like "--repository" to be shared across multiple action parsers. + Given a sequence of arguments, a dict from action name to argparse.ArgumentParser instance, + and the global parser as a argparse.ArgumentParser instance, give each requested action's + parser a shot at parsing all arguments. This allows common arguments like "--repository" to be + shared across multiple action parsers. Return the result as a tuple of: (a dict mapping from action name to an argparse.Namespace of - parsed arguments, a list of strings of remaining arguments not claimed by any action parser). + parsed arguments, a tuple of argument tuples where each is the remaining arguments not claimed + by any action parser). ''' arguments = collections.OrderedDict() help_requested = bool('--help' in unparsed_arguments or '-h' in unparsed_arguments) @@ -211,11 +213,12 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers): ) ) + arguments['global'], remaining = global_parser.parse_known_args(unparsed_arguments) + remaining_action_arguments.append(remaining) + return ( arguments, - get_unparsable_arguments(tuple(remaining_action_arguments)) - if arguments - else unparsed_arguments, + tuple(remaining_action_arguments) if arguments else unparsed_arguments, ) @@ -235,7 +238,10 @@ class Extend_action(Action): def make_parsers(): ''' - Build a top-level parser and its action parsers and return them as a tuple. + Build a global arguments parser, individual action parsers, and a combined parser containing + both. Return them as a tuple. The global parser is useful for parsing just global arguments + while ignoring actions, and the combined parser is handy for displaying help that includes + everything: global flags, a list of actions, etc. ''' config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) @@ -345,7 +351,7 @@ def make_parsers(): help='Display installed version number of borgmatic and exit', ) - top_level_parser = ArgumentParser( + global_plus_action_parser = ArgumentParser( description=''' Simple, configuration-driven backup software for servers and workstations. If none of the action options are given, then borgmatic defaults to: create, prune, compact, and @@ -354,7 +360,7 @@ def make_parsers(): parents=[global_parser], ) - action_parsers = top_level_parser.add_subparsers( + action_parsers = global_plus_action_parser.add_subparsers( title='actions', metavar='', help='Specify zero or more actions. Defaults to create, prune, compact, and check. Use --help with action for details:', @@ -776,7 +782,7 @@ def make_parsers(): config_validate_parser = config_parsers.add_parser( 'validate', - help='Validate that borgmatic configuration files specified with --config (see borgmatic --help)', + help='Validate borgmatic configuration files specified with --config (see borgmatic --help)', description='Validate borgmatic configuration files specified with --config (see borgmatic --help)', add_help=False, ) @@ -1221,27 +1227,46 @@ def make_parsers(): ) borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - return top_level_parser, action_parsers + return global_parser, action_parsers, global_plus_action_parser def parse_arguments(*unparsed_arguments): ''' Given command-line arguments with which this script was invoked, parse the arguments and return them as a dict mapping from action name (or "global") to an argparse.Namespace instance. - ''' - top_level_parser, action_parsers = make_parsers() - arguments, remaining_arguments = parse_arguments_for_actions( - unparsed_arguments, action_parsers.choices + Raise ValueError if the arguments cannot be parsed. + Raise SystemExit with an error code of 0 if "--help" was requested. + ''' + global_parser, action_parsers, global_plus_action_parser = make_parsers() + arguments, remaining_action_arguments = parse_arguments_for_actions( + unparsed_arguments, action_parsers.choices, global_parser ) for action_name in ('bootstrap', 'generate', 'validate'): - if action_name in arguments.keys() and len(arguments.keys()) > 1: + if ( + action_name in arguments.keys() and len(arguments.keys()) > 2 + ): # 2 = 1 for 'global' + 1 for the action raise ValueError( - 'The {action_name} action cannot be combined with other actions. Please run it separately.' + f'The {action_name} action cannot be combined with other actions. Please run it separately.' ) - arguments['global'] = top_level_parser.parse_args(remaining_arguments) + unknown_arguments = get_unparsable_arguments(remaining_action_arguments) + + if unknown_arguments: + if '--help' in unknown_arguments or '-h' in unknown_arguments: + global_plus_action_parser.print_help() + sys.exit(0) + + global_plus_action_parser.print_usage() + raise ValueError( + f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}" + ) + + # Prevent action names that follow "--config" paths from being considered as additional paths. + for argument_name in arguments.keys(): + if argument_name != 'global' and argument_name in arguments['global'].config_paths: + arguments['global'].config_paths.remove(argument_name) if arguments['global'].excludes_filename: raise ValueError( diff --git a/borgmatic/commands/completion/bash.py b/borgmatic/commands/completion/bash.py index d20eca42..7bf28a42 100644 --- a/borgmatic/commands/completion/bash.py +++ b/borgmatic/commands/completion/bash.py @@ -15,8 +15,12 @@ def bash_completion(): Return a bash completion script for the borgmatic command. Produce this by introspecting borgmatic's command-line argument parsers. ''' - top_level_parser, subparsers = borgmatic.commands.arguments.make_parsers() - global_flags = parser_flags(top_level_parser) + ( + unused_global_parser, + action_parsers, + global_plus_action_parser, + ) = borgmatic.commands.arguments.make_parsers() + global_flags = parser_flags(global_plus_action_parser) # Avert your eyes. return '\n'.join( @@ -41,18 +45,18 @@ def bash_completion(): fi''' % ( action, - parser_flags(subparser), + parser_flags(action_parser), ' '.join( - borgmatic.commands.completion.actions.available_actions(subparsers, action) + borgmatic.commands.completion.actions.available_actions(action_parsers, action) ), global_flags, ) - for action, subparser in reversed(subparsers.choices.items()) + for action, action_parser in reversed(action_parsers.choices.items()) ) + ( ' COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))' # noqa: FS003 % ( - ' '.join(borgmatic.commands.completion.actions.available_actions(subparsers)), + ' '.join(borgmatic.commands.completion.actions.available_actions(action_parsers)), global_flags, ), ' (check_version &)', diff --git a/borgmatic/commands/completion/fish.py b/borgmatic/commands/completion/fish.py index 599617ce..edca0226 100644 --- a/borgmatic/commands/completion/fish.py +++ b/borgmatic/commands/completion/fish.py @@ -91,18 +91,22 @@ def fish_completion(): Return a fish completion script for the borgmatic command. Produce this by introspecting borgmatic's command-line argument parsers. ''' - top_level_parser, subparsers = borgmatic.commands.arguments.make_parsers() + ( + unused_global_parser, + action_parsers, + global_plus_action_parser, + ) = borgmatic.commands.arguments.make_parsers() - all_subparsers = ' '.join(action for action in subparsers.choices.keys()) + all_action_parsers = ' '.join(action for action in action_parsers.choices.keys()) exact_option_args = tuple( ' '.join(action.option_strings) - for subparser in subparsers.choices.values() - for action in subparser._actions + for action_parser in action_parsers.choices.values() + for action in action_parser._actions if has_exact_options(action) ) + tuple( ' '.join(action.option_strings) - for action in top_level_parser._actions + for action in global_plus_action_parser._actions if len(action.option_strings) > 0 if has_exact_options(action) ) @@ -144,29 +148,29 @@ def fish_completion(): return 1 end - set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}" + set --local action_parser_condition "not __fish_seen_subcommand_from {all_action_parsers}" set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}" ''' ) - + ('\n# subparser completions',) + + ('\n# action_parser completions',) + tuple( - f'''complete -c borgmatic -f -n "$subparser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' - for action_name, subparser in subparsers.choices.items() + f'''complete -c borgmatic -f -n "$action_parser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(action_parser.description)}''' + for action_name, action_parser in action_parsers.choices.items() ) + ('\n# global flags',) + tuple( # -n is checked in order, so put faster / more likely to be true checks first f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' - for action in top_level_parser._actions + for action in global_plus_action_parser._actions # ignore the noargs action, as this is an impossible completion for fish if len(action.option_strings) > 0 if 'Deprecated' not in action.help ) - + ('\n# subparser flags',) + + ('\n# action_parser flags',) + tuple( f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}''' - for action_name, subparser in subparsers.choices.items() - for action in subparser._actions + for action_name, action_parser in action_parsers.choices.items() + for action in action_parser._actions if 'Deprecated' not in (action.help or ()) ) ) diff --git a/tests/integration/commands/completion/test_actions.py b/tests/integration/commands/completion/test_actions.py index 1a8d69e6..2e6fde9b 100644 --- a/tests/integration/commands/completion/test_actions.py +++ b/tests/integration/commands/completion/test_actions.py @@ -3,18 +3,26 @@ from borgmatic.commands.completion import actions as module def test_available_actions_uses_only_subactions_for_action_with_subactions(): - unused_top_level_parser, subparsers = borgmatic.commands.arguments.make_parsers() + ( + unused_global_parser, + action_parsers, + unused_combined_parser, + ) = borgmatic.commands.arguments.make_parsers() - actions = module.available_actions(subparsers, 'config') + actions = module.available_actions(action_parsers, 'config') assert 'bootstrap' in actions assert 'list' not in actions def test_available_actions_omits_subactions_for_action_without_subactions(): - unused_top_level_parser, subparsers = borgmatic.commands.arguments.make_parsers() + ( + unused_global_parser, + action_parsers, + unused_combined_parser, + ) = borgmatic.commands.arguments.make_parsers() - actions = module.available_actions(subparsers, 'list') + actions = module.available_actions(action_parsers, 'list') assert 'bootstrap' not in actions assert 'config' in actions diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 47ec72fa..89141215 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -30,6 +30,17 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list(): assert global_arguments.log_file_verbosity == 0 +def test_parse_arguments_with_action_after_config_path_omits_action(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + arguments = module.parse_arguments('--config', 'myconfig', 'list', '--json') + + global_arguments = arguments['global'] + assert global_arguments.config_paths == ['myconfig'] + assert 'list' in arguments + assert arguments['list'].json + + def test_parse_arguments_with_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) @@ -194,10 +205,10 @@ def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled(): assert 'check' in arguments -def test_parse_arguments_with_invalid_arguments_exits(): +def test_parse_arguments_disallows_invalid_argument(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): + with pytest.raises(ValueError): module.parse_arguments('--posix-me-harder') @@ -211,7 +222,7 @@ def test_parse_arguments_disallows_deprecated_excludes_option(): def test_parse_arguments_disallows_encryption_mode_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): + with pytest.raises(ValueError): module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey') @@ -231,14 +242,14 @@ def test_parse_arguments_requires_encryption_mode_with_init(): def test_parse_arguments_disallows_append_only_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): + with pytest.raises(ValueError): module.parse_arguments('--config', 'myconfig', '--append-only') def test_parse_arguments_disallows_storage_quota_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): + with pytest.raises(ValueError): module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G') @@ -287,14 +298,14 @@ def test_parse_arguments_allows_repository_with_list(): def test_parse_arguments_disallows_archive_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): + with pytest.raises(ValueError): module.parse_arguments('--config', 'myconfig', '--archive', 'test') def test_parse_arguments_disallows_paths_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): + with pytest.raises(ValueError): module.parse_arguments('--config', 'myconfig', '--path', 'test') @@ -380,7 +391,7 @@ def test_parse_arguments_allows_progress_and_extract(): def test_parse_arguments_disallows_progress_without_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): + with pytest.raises(ValueError): module.parse_arguments('--progress', 'list') @@ -399,7 +410,7 @@ def test_parse_arguments_with_stats_and_prune_flags_does_not_raise(): def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): + with pytest.raises(ValueError): module.parse_arguments('--stats', 'list') @@ -535,11 +546,9 @@ def test_parse_arguments_extract_with_check_only_extract_does_not_raise(): def test_parse_arguments_bootstrap_without_config_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit) as exit: + with pytest.raises(ValueError): module.parse_arguments('bootstrap') - assert exit.value.code == 2 - def test_parse_arguments_config_with_no_subaction_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 3c32a7fa..4a7ef432 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -130,6 +130,175 @@ def test_parse_and_record_action_arguments_with_borg_action_consumes_arguments_a assert borg_parsed_arguments.options == ('list',) +@pytest.mark.parametrize( + 'arguments, expected', + [ + # A global flag remaining from each parsed action. + ( + ( + ('--latest', 'archive', 'prune', 'extract', 'list', '--test-flag'), + ('--latest', 'archive', 'check', 'extract', 'list', '--test-flag'), + ('prune', 'check', 'list', '--test-flag'), + ('prune', 'check', 'extract', '--test-flag'), + ), + ('--test-flag',), + ), + # No global flags remaining. + ( + ( + ('--latest', 'archive', 'prune', 'extract', 'list'), + ('--latest', 'archive', 'check', 'extract', 'list'), + ('prune', 'check', 'list'), + ('prune', 'check', 'extract'), + ), + (), + ), + # Multiple of the same value across global flags. + ( + ( + ('--verbosity', '2', '--syslog-verbosity', '2', '--monitoring-verbosity', '2'), + ('--verbosity', '2', '--syslog-verbosity', '2', '--monitoring-verbosity', '2'), + ), + ('--verbosity', '2', '--syslog-verbosity', '2', '--monitoring-verbosity', '2'), + ), + # Multiple of the same value across action and global flags. + ( + ( + ('list', '--archive', 'test', '--log-file', 'test'), + ('prune', '--log-file', 'test'), + ), + ('--log-file', 'test'), + ), + # No flags. + ((), ()), + ], +) +def test_get_unparsable_arguments_returns_remaining_arguments_that_no_action_can_parse( + arguments, expected +): + assert module.get_unparsable_arguments(arguments) == expected + + +def test_get_subaction_parsers_with_subactions_returns_one_entry_per_subaction(): + foo_parser = flexmock() + bar_parser = flexmock() + baz_parser = flexmock() + + assert module.get_subaction_parsers( + flexmock( + _subparsers=flexmock( + _group_actions=( + flexmock(choices={'foo': foo_parser, 'bar': bar_parser}), + flexmock(choices={'baz': baz_parser}), + ) + ) + ) + ) == {'foo': foo_parser, 'bar': bar_parser, 'baz': baz_parser} + + +def test_get_subactions_for_actions_with_no_subactions_returns_empty_result(): + assert module.get_subactions_for_actions({'action': flexmock(_subparsers=None)}) == {} + + +def test_get_subactions_for_actions_with_subactions_returns_one_entry_per_action(): + assert module.get_subactions_for_actions( + { + 'action': flexmock( + _subparsers=flexmock( + _group_actions=( + flexmock(choices={'foo': flexmock(), 'bar': flexmock()}), + flexmock(choices={'baz': flexmock()}), + ) + ) + ), + 'other': flexmock( + _subparsers=flexmock(_group_actions=(flexmock(choices={'quux': flexmock()}),)) + ), + } + ) == {'action': ('foo', 'bar', 'baz'), 'other': ('quux',)} + + +def test_omit_values_colliding_with_action_names_drops_action_names_that_have__been_parsed_as_values(): + assert module.omit_values_colliding_with_action_names( + ('check', '--only', 'extract', '--some-list', 'borg'), + {'check': flexmock(only='extract', some_list=['borg'])}, + ) == ('check', '--only', '--some-list') + + +def test_parse_and_record_action_arguments_without_action_name_leaves_arguments_untouched(): + unparsed_arguments = ('--foo', '--bar') + flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( + unparsed_arguments + ) + + assert ( + module.parse_and_record_action_arguments( + unparsed_arguments, flexmock(), flexmock(), 'action' + ) + == unparsed_arguments + ) + + +def test_parse_and_record_action_arguments_updates_parsed_arguments_and_returns_remaining(): + unparsed_arguments = ('action', '--foo', '--bar', '--verbosity', '1') + other_parsed_arguments = flexmock() + parsed_arguments = {'other': other_parsed_arguments} + action_parsed_arguments = flexmock() + flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( + unparsed_arguments + ) + action_parser = flexmock() + flexmock(action_parser).should_receive('parse_known_args').and_return( + action_parsed_arguments, ('action', '--verbosity', '1') + ) + + assert module.parse_and_record_action_arguments( + unparsed_arguments, parsed_arguments, action_parser, 'action' + ) == ('--verbosity', '1') + assert parsed_arguments == {'other': other_parsed_arguments, 'action': action_parsed_arguments} + + +def test_parse_and_record_action_arguments_with_alias_updates_canonical_parsed_arguments(): + unparsed_arguments = ('action', '--foo', '--bar', '--verbosity', '1') + other_parsed_arguments = flexmock() + parsed_arguments = {'other': other_parsed_arguments} + action_parsed_arguments = flexmock() + flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( + unparsed_arguments + ) + action_parser = flexmock() + flexmock(action_parser).should_receive('parse_known_args').and_return( + action_parsed_arguments, ('action', '--verbosity', '1') + ) + + assert module.parse_and_record_action_arguments( + unparsed_arguments, parsed_arguments, action_parser, 'action', canonical_name='doit' + ) == ('--verbosity', '1') + assert parsed_arguments == {'other': other_parsed_arguments, 'doit': action_parsed_arguments} + + +def test_parse_and_record_action_arguments_with_borg_action_consumes_arguments_after_action_name(): + unparsed_arguments = ('--verbosity', '1', 'borg', 'list') + parsed_arguments = {} + borg_parsed_arguments = flexmock(options=flexmock()) + flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( + unparsed_arguments + ) + borg_parser = flexmock() + flexmock(borg_parser).should_receive('parse_known_args').and_return( + borg_parsed_arguments, ('--verbosity', '1', 'borg', 'list') + ) + + assert module.parse_and_record_action_arguments( + unparsed_arguments, + parsed_arguments, + borg_parser, + 'borg', + ) == ('--verbosity', '1') + assert parsed_arguments == {'borg': borg_parsed_arguments} + assert borg_parsed_arguments.options == ('list',) + + @pytest.mark.parametrize( 'arguments, expected', [ @@ -167,63 +336,74 @@ def test_parse_arguments_for_actions_consumes_action_arguments_before_action_nam lambda unparsed, parsed, parser, action, canonical=None: parsed.update( {action: action_namespace} ) + or () ).and_return(()) flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - flexmock(module).should_receive('get_unparsable_arguments').and_return(()) action_parsers = {'action': flexmock(), 'other': flexmock()} + global_namespace = flexmock() + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) - arguments, remaining_arguments = module.parse_arguments_for_actions( - ('--foo', 'true', 'action'), action_parsers + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('--foo', 'true', 'action'), action_parsers, global_parser ) - assert arguments == {'action': action_namespace} - assert remaining_arguments == () + assert arguments == {'global': global_namespace, 'action': action_namespace} + assert remaining_action_arguments == ((), ()) def test_parse_arguments_for_actions_consumes_action_arguments_after_action_name(): action_namespace = flexmock(foo=True) + remaining = flexmock() flexmock(module).should_receive('get_subaction_parsers').and_return({}) flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( lambda unparsed, parsed, parser, action, canonical=None: parsed.update( {action: action_namespace} ) - ).and_return(()) + or remaining + ) flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - flexmock(module).should_receive('get_unparsable_arguments').and_return(()) action_parsers = {'action': flexmock(), 'other': flexmock()} + global_namespace = flexmock() + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) - arguments, remaining_arguments = module.parse_arguments_for_actions( - ('action', '--foo', 'true'), action_parsers + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('action', '--foo', 'true'), action_parsers, global_parser ) - assert arguments == {'action': action_namespace} - assert remaining_arguments == () + assert arguments == {'global': global_namespace, 'action': action_namespace} + assert remaining_action_arguments == (remaining, ()) def test_parse_arguments_for_actions_consumes_action_arguments_with_alias(): action_namespace = flexmock(foo=True) + remaining = flexmock() flexmock(module).should_receive('get_subaction_parsers').and_return({}) flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( lambda unparsed, parsed, parser, action, canonical=None: parsed.update( {canonical or action: action_namespace} ) - ).and_return(()) + or remaining + ) flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - flexmock(module).should_receive('get_unparsable_arguments').and_return(()) action_parsers = { 'action': flexmock(), '-a': flexmock(), 'other': flexmock(), '-o': flexmock(), } + global_namespace = flexmock() + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) flexmock(module).ACTION_ALIASES = {'action': ['-a'], 'other': ['-o']} - arguments, remaining_arguments = module.parse_arguments_for_actions( - ('-a', '--foo', 'true'), action_parsers + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('-a', '--foo', 'true'), action_parsers, global_parser ) - assert arguments == {'action': action_namespace} - assert remaining_arguments == () + assert arguments == {'global': global_namespace, 'action': action_namespace} + assert remaining_action_arguments == (remaining, ()) def test_parse_arguments_for_actions_consumes_multiple_action_arguments(): @@ -234,20 +414,27 @@ def test_parse_arguments_for_actions_consumes_multiple_action_arguments(): lambda unparsed, parsed, parser, action, canonical=None: parsed.update( {action: action_namespace if action == 'action' else other_namespace} ) + or () ).and_return(('other', '--bar', '3')).and_return('action', '--foo', 'true') flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - flexmock(module).should_receive('get_unparsable_arguments').and_return(()) action_parsers = { 'action': flexmock(), 'other': flexmock(), } + global_namespace = flexmock() + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) - arguments, remaining_arguments = module.parse_arguments_for_actions( - ('action', '--foo', 'true', 'other', '--bar', '3'), action_parsers + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('action', '--foo', 'true', 'other', '--bar', '3'), action_parsers, global_parser ) - assert arguments == {'action': action_namespace, 'other': other_namespace} - assert remaining_arguments == () + assert arguments == { + 'global': global_namespace, + 'action': action_namespace, + 'other': other_namespace, + } + assert remaining_action_arguments == ((), (), ()) def test_parse_arguments_for_actions_respects_command_line_action_ordering(): @@ -258,26 +445,31 @@ def test_parse_arguments_for_actions_respects_command_line_action_ordering(): lambda unparsed, parsed, parser, action, canonical=None: parsed.update( {action: other_namespace if action == 'other' else action_namespace} ) + or () ).and_return(('action',)).and_return(('other', '--foo', 'true')) flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - flexmock(module).should_receive('get_unparsable_arguments').and_return(()) action_parsers = { 'action': flexmock(), 'other': flexmock(), } + global_namespace = flexmock() + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) - arguments, remaining_arguments = module.parse_arguments_for_actions( - ('other', '--foo', 'true', 'action'), action_parsers + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('other', '--foo', 'true', 'action'), action_parsers, global_parser ) assert arguments == collections.OrderedDict( - [('other', other_namespace), ('action', action_namespace)] + [('other', other_namespace), ('action', action_namespace), ('global', global_namespace)] ) - assert remaining_arguments == () + assert remaining_action_arguments == ((), (), ()) def test_parse_arguments_for_actions_applies_default_action_parsers(): + global_namespace = flexmock() namespaces = { + 'global': global_namespace, 'prune': flexmock(), 'compact': flexmock(), 'create': flexmock(progress=True), @@ -289,9 +481,9 @@ def test_parse_arguments_for_actions_applies_default_action_parsers(): lambda unparsed, parsed, parser, action, canonical=None: parsed.update( {action: namespaces.get(action)} ) + or () ).and_return(()) flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - flexmock(module).should_receive('get_unparsable_arguments').and_return(()) action_parsers = { 'prune': flexmock(), 'compact': flexmock(), @@ -299,13 +491,41 @@ def test_parse_arguments_for_actions_applies_default_action_parsers(): 'check': flexmock(), 'other': flexmock(), } + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) - arguments, remaining_arguments = module.parse_arguments_for_actions( - ('--progress'), action_parsers + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('--progress'), action_parsers, global_parser ) assert arguments == namespaces - assert remaining_arguments == () + assert remaining_action_arguments == ((), (), (), (), ()) + + +def test_parse_arguments_for_actions_consumes_global_arguments(): + action_namespace = flexmock() + flexmock(module).should_receive('get_subaction_parsers').and_return({}) + flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( + lambda unparsed, parsed, parser, action, canonical=None: parsed.update( + {action: action_namespace} + ) + or ('--verbosity', 'lots') + ) + flexmock(module).should_receive('get_subactions_for_actions').and_return({}) + action_parsers = { + 'action': flexmock(), + 'other': flexmock(), + } + global_namespace = flexmock() + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) + + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('action', '--verbosity', 'lots'), action_parsers, global_parser + ) + + assert arguments == {'global': global_namespace, 'action': action_namespace} + assert remaining_action_arguments == (('--verbosity', 'lots'), ()) def test_parse_arguments_for_actions_passes_through_unknown_arguments_before_action_name(): @@ -315,20 +535,23 @@ def test_parse_arguments_for_actions_passes_through_unknown_arguments_before_act lambda unparsed, parsed, parser, action, canonical=None: parsed.update( {action: action_namespace} ) - ).and_return(('--verbosity', 'lots')) + or ('--wtf', 'yes') + ) flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - flexmock(module).should_receive('get_unparsable_arguments').and_return(('--verbosity', 'lots')) action_parsers = { 'action': flexmock(), 'other': flexmock(), } + global_namespace = flexmock() + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) - arguments, remaining_arguments = module.parse_arguments_for_actions( - ('--verbosity', 'lots', 'action'), action_parsers + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('--wtf', 'yes', 'action'), action_parsers, global_parser ) - assert arguments == {'action': action_namespace} - assert remaining_arguments == ('--verbosity', 'lots') + assert arguments == {'global': global_namespace, 'action': action_namespace} + assert remaining_action_arguments == (('--wtf', 'yes'), ()) def test_parse_arguments_for_actions_passes_through_unknown_arguments_after_action_name(): @@ -338,20 +561,23 @@ def test_parse_arguments_for_actions_passes_through_unknown_arguments_after_acti lambda unparsed, parsed, parser, action, canonical=None: parsed.update( {action: action_namespace} ) - ).and_return(('--verbosity', 'lots')) + or ('--wtf', 'yes') + ) flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - flexmock(module).should_receive('get_unparsable_arguments').and_return(('--verbosity', 'lots')) action_parsers = { 'action': flexmock(), 'other': flexmock(), } + global_namespace = flexmock() + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) - arguments, remaining_arguments = module.parse_arguments_for_actions( - ('action', '--verbosity', 'lots'), action_parsers + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('action', '--wtf', 'yes'), action_parsers, global_parser ) - assert arguments == {'action': action_namespace} - assert remaining_arguments == ('--verbosity', 'lots') + assert arguments == {'global': global_namespace, 'action': action_namespace} + assert remaining_action_arguments == (('--wtf', 'yes'), ()) def test_parse_arguments_for_actions_with_borg_action_skips_other_action_parsers(): @@ -361,20 +587,23 @@ def test_parse_arguments_for_actions_with_borg_action_skips_other_action_parsers lambda unparsed, parsed, parser, action, canonical=None: parsed.update( {action: action_namespace} ) + or () ).and_return(()) flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - flexmock(module).should_receive('get_unparsable_arguments').and_return(()) action_parsers = { 'borg': flexmock(), 'list': flexmock(), } + global_namespace = flexmock() + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) - arguments, remaining_arguments = module.parse_arguments_for_actions( - ('borg', 'list'), action_parsers + arguments, remaining_action_arguments = module.parse_arguments_for_actions( + ('borg', 'list'), action_parsers, global_parser ) - assert arguments == {'borg': action_namespace} - assert remaining_arguments == () + assert arguments == {'global': global_namespace, 'borg': action_namespace} + assert remaining_action_arguments == ((), ()) def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): @@ -384,6 +613,8 @@ def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): {'config': ['bootstrap']} ) action_parsers = {'config': flexmock()} + global_parser = flexmock() + global_parser.should_receive('parse_known_args').and_return((flexmock(), ())) with pytest.raises(ValueError): - module.parse_arguments_for_actions(('config',), action_parsers) + module.parse_arguments_for_actions(('config',), action_parsers, global_parser) From 8debcbeabad115c52055d2b012b7018c254fbd9b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 24 Jun 2023 14:28:50 -0700 Subject: [PATCH 301/344] Remove duplicated tests (#716). --- tests/unit/commands/test_arguments.py | 189 -------------------------- 1 file changed, 189 deletions(-) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 4a7ef432..0969a6a5 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -153,22 +153,6 @@ def test_parse_and_record_action_arguments_with_borg_action_consumes_arguments_a ), (), ), - # Multiple of the same value across global flags. - ( - ( - ('--verbosity', '2', '--syslog-verbosity', '2', '--monitoring-verbosity', '2'), - ('--verbosity', '2', '--syslog-verbosity', '2', '--monitoring-verbosity', '2'), - ), - ('--verbosity', '2', '--syslog-verbosity', '2', '--monitoring-verbosity', '2'), - ), - # Multiple of the same value across action and global flags. - ( - ( - ('list', '--archive', 'test', '--log-file', 'test'), - ('prune', '--log-file', 'test'), - ), - ('--log-file', 'test'), - ), # No flags. ((), ()), ], @@ -179,179 +163,6 @@ def test_get_unparsable_arguments_returns_remaining_arguments_that_no_action_can assert module.get_unparsable_arguments(arguments) == expected -def test_get_subaction_parsers_with_subactions_returns_one_entry_per_subaction(): - foo_parser = flexmock() - bar_parser = flexmock() - baz_parser = flexmock() - - assert module.get_subaction_parsers( - flexmock( - _subparsers=flexmock( - _group_actions=( - flexmock(choices={'foo': foo_parser, 'bar': bar_parser}), - flexmock(choices={'baz': baz_parser}), - ) - ) - ) - ) == {'foo': foo_parser, 'bar': bar_parser, 'baz': baz_parser} - - -def test_get_subactions_for_actions_with_no_subactions_returns_empty_result(): - assert module.get_subactions_for_actions({'action': flexmock(_subparsers=None)}) == {} - - -def test_get_subactions_for_actions_with_subactions_returns_one_entry_per_action(): - assert module.get_subactions_for_actions( - { - 'action': flexmock( - _subparsers=flexmock( - _group_actions=( - flexmock(choices={'foo': flexmock(), 'bar': flexmock()}), - flexmock(choices={'baz': flexmock()}), - ) - ) - ), - 'other': flexmock( - _subparsers=flexmock(_group_actions=(flexmock(choices={'quux': flexmock()}),)) - ), - } - ) == {'action': ('foo', 'bar', 'baz'), 'other': ('quux',)} - - -def test_omit_values_colliding_with_action_names_drops_action_names_that_have__been_parsed_as_values(): - assert module.omit_values_colliding_with_action_names( - ('check', '--only', 'extract', '--some-list', 'borg'), - {'check': flexmock(only='extract', some_list=['borg'])}, - ) == ('check', '--only', '--some-list') - - -def test_parse_and_record_action_arguments_without_action_name_leaves_arguments_untouched(): - unparsed_arguments = ('--foo', '--bar') - flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( - unparsed_arguments - ) - - assert ( - module.parse_and_record_action_arguments( - unparsed_arguments, flexmock(), flexmock(), 'action' - ) - == unparsed_arguments - ) - - -def test_parse_and_record_action_arguments_updates_parsed_arguments_and_returns_remaining(): - unparsed_arguments = ('action', '--foo', '--bar', '--verbosity', '1') - other_parsed_arguments = flexmock() - parsed_arguments = {'other': other_parsed_arguments} - action_parsed_arguments = flexmock() - flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( - unparsed_arguments - ) - action_parser = flexmock() - flexmock(action_parser).should_receive('parse_known_args').and_return( - action_parsed_arguments, ('action', '--verbosity', '1') - ) - - assert module.parse_and_record_action_arguments( - unparsed_arguments, parsed_arguments, action_parser, 'action' - ) == ('--verbosity', '1') - assert parsed_arguments == {'other': other_parsed_arguments, 'action': action_parsed_arguments} - - -def test_parse_and_record_action_arguments_with_alias_updates_canonical_parsed_arguments(): - unparsed_arguments = ('action', '--foo', '--bar', '--verbosity', '1') - other_parsed_arguments = flexmock() - parsed_arguments = {'other': other_parsed_arguments} - action_parsed_arguments = flexmock() - flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( - unparsed_arguments - ) - action_parser = flexmock() - flexmock(action_parser).should_receive('parse_known_args').and_return( - action_parsed_arguments, ('action', '--verbosity', '1') - ) - - assert module.parse_and_record_action_arguments( - unparsed_arguments, parsed_arguments, action_parser, 'action', canonical_name='doit' - ) == ('--verbosity', '1') - assert parsed_arguments == {'other': other_parsed_arguments, 'doit': action_parsed_arguments} - - -def test_parse_and_record_action_arguments_with_borg_action_consumes_arguments_after_action_name(): - unparsed_arguments = ('--verbosity', '1', 'borg', 'list') - parsed_arguments = {} - borg_parsed_arguments = flexmock(options=flexmock()) - flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( - unparsed_arguments - ) - borg_parser = flexmock() - flexmock(borg_parser).should_receive('parse_known_args').and_return( - borg_parsed_arguments, ('--verbosity', '1', 'borg', 'list') - ) - - assert module.parse_and_record_action_arguments( - unparsed_arguments, - parsed_arguments, - borg_parser, - 'borg', - ) == ('--verbosity', '1') - assert parsed_arguments == {'borg': borg_parsed_arguments} - assert borg_parsed_arguments.options == ('list',) - - -@pytest.mark.parametrize( - 'arguments, expected', - [ - ( - ( - ('--latest', 'archive', 'prune', 'extract', 'list', '--test-flag'), - ('--latest', 'archive', 'check', 'extract', 'list', '--test-flag'), - ('prune', 'check', 'list', '--test-flag'), - ('prune', 'check', 'extract', '--test-flag'), - ), - ('--test-flag',), - ), - ( - ( - ('--latest', 'archive', 'prune', 'extract', 'list'), - ('--latest', 'archive', 'check', 'extract', 'list'), - ('prune', 'check', 'list'), - ('prune', 'check', 'extract'), - ), - (), - ), - ((), ()), - ], -) -def test_get_unparsable_arguments_returns_remaining_arguments_that_no_action_can_parse( - arguments, expected -): - assert module.get_unparsable_arguments(arguments) == expected - - -def test_parse_arguments_for_actions_consumes_action_arguments_before_action_name(): - action_namespace = flexmock(foo=True) - flexmock(module).should_receive('get_subaction_parsers').and_return({}) - flexmock(module).should_receive('parse_and_record_action_arguments').replace_with( - lambda unparsed, parsed, parser, action, canonical=None: parsed.update( - {action: action_namespace} - ) - or () - ).and_return(()) - flexmock(module).should_receive('get_subactions_for_actions').and_return({}) - action_parsers = {'action': flexmock(), 'other': flexmock()} - global_namespace = flexmock() - global_parser = flexmock() - global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) - - arguments, remaining_action_arguments = module.parse_arguments_for_actions( - ('--foo', 'true', 'action'), action_parsers, global_parser - ) - - assert arguments == {'global': global_namespace, 'action': action_namespace} - assert remaining_action_arguments == ((), ()) - - def test_parse_arguments_for_actions_consumes_action_arguments_after_action_name(): action_namespace = flexmock(foo=True) remaining = flexmock() From b62017be4bbeae110f813d3bece3f51b49740522 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 24 Jun 2023 15:35:10 -0700 Subject: [PATCH 302/344] Fix edge case in which "--config somepath.yaml" followed by an action alias (e.g. init for rcreate) wasn't parsed correctly (#716). --- borgmatic/commands/arguments.py | 15 ++++++++++----- tests/end-to-end/test_borgmatic.py | 2 +- tests/end-to-end/test_generate_config.py | 4 ++-- tests/end-to-end/test_override.py | 2 +- tests/end-to-end/test_validate_config.py | 6 +++--- tests/integration/commands/test_arguments.py | 11 +++++++++++ tests/unit/commands/test_arguments.py | 18 +++++++++--------- 7 files changed, 37 insertions(+), 21 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index eac88c8a..6b857f2c 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -216,6 +216,16 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse arguments['global'], remaining = global_parser.parse_known_args(unparsed_arguments) remaining_action_arguments.append(remaining) + # Prevent action names that follow "--config" paths from being considered as additional paths. + for argument_name in arguments.keys(): + if argument_name == 'global': + continue + + for action_name in [argument_name] + ACTION_ALIASES.get(argument_name, []): + if action_name in arguments['global'].config_paths: + arguments['global'].config_paths.remove(action_name) + break + return ( arguments, tuple(remaining_action_arguments) if arguments else unparsed_arguments, @@ -1263,11 +1273,6 @@ def parse_arguments(*unparsed_arguments): f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}" ) - # Prevent action names that follow "--config" paths from being considered as additional paths. - for argument_name in arguments.keys(): - if argument_name != 'global' and argument_name in arguments['global'].config_paths: - arguments['global'].config_paths.remove(argument_name) - if arguments['global'].excludes_filename: raise ValueError( 'The --excludes flag has been replaced with exclude_patterns in configuration.' diff --git a/tests/end-to-end/test_borgmatic.py b/tests/end-to-end/test_borgmatic.py index 5e915f00..f2b81a81 100644 --- a/tests/end-to-end/test_borgmatic.py +++ b/tests/end-to-end/test_borgmatic.py @@ -12,7 +12,7 @@ def generate_configuration(config_path, repository_path): to work for testing (including injecting the given repository path and tacking on an encryption passphrase). ''' - subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) config = ( open(config_path) .read() diff --git a/tests/end-to-end/test_generate_config.py b/tests/end-to-end/test_generate_config.py index b8cade96..c9293b7e 100644 --- a/tests/end-to-end/test_generate_config.py +++ b/tests/end-to-end/test_generate_config.py @@ -8,9 +8,9 @@ def test_generate_borgmatic_config_with_merging_succeeds(): config_path = os.path.join(temporary_directory, 'test.yaml') new_config_path = os.path.join(temporary_directory, 'new.yaml') - subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) subprocess.check_call( - f'generate-borgmatic-config --source {config_path} --destination {new_config_path}'.split( + f'borgmatic config generate --source {config_path} --destination {new_config_path}'.split( ' ' ) ) diff --git a/tests/end-to-end/test_override.py b/tests/end-to-end/test_override.py index e86186d9..e91e28bf 100644 --- a/tests/end-to-end/test_override.py +++ b/tests/end-to-end/test_override.py @@ -10,7 +10,7 @@ def generate_configuration(config_path, repository_path): to work for testing (including injecting the given repository path and tacking on an encryption passphrase). ''' - subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) config = ( open(config_path) .read() diff --git a/tests/end-to-end/test_validate_config.py b/tests/end-to-end/test_validate_config.py index 54465033..4b86da4a 100644 --- a/tests/end-to-end/test_validate_config.py +++ b/tests/end-to-end/test_validate_config.py @@ -8,7 +8,7 @@ def test_validate_config_command_with_valid_configuration_succeeds(): with tempfile.TemporaryDirectory() as temporary_directory: config_path = os.path.join(temporary_directory, 'test.yaml') - subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' ')) assert exit_code == 0 @@ -18,7 +18,7 @@ def test_validate_config_command_with_invalid_configuration_fails(): with tempfile.TemporaryDirectory() as temporary_directory: config_path = os.path.join(temporary_directory, 'test.yaml') - subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) config = open(config_path).read().replace('keep_daily: 7', 'keep_daily: "7"') config_file = open(config_path, 'w') config_file.write(config) @@ -33,7 +33,7 @@ def test_validate_config_command_with_show_flag_displays_configuration(): with tempfile.TemporaryDirectory() as temporary_directory: config_path = os.path.join(temporary_directory, 'test.yaml') - subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) output = subprocess.check_output( f'validate-borgmatic-config --config {config_path} --show'.split(' ') ).decode(sys.stdout.encoding) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 89141215..f992f9aa 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -41,6 +41,17 @@ def test_parse_arguments_with_action_after_config_path_omits_action(): assert arguments['list'].json +def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + arguments = module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey') + + global_arguments = arguments['global'] + assert global_arguments.config_paths == ['myconfig'] + assert 'rcreate' in arguments + assert arguments['rcreate'].encryption_mode == 'repokey' + + def test_parse_arguments_with_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 0969a6a5..55c0eef4 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -175,7 +175,7 @@ def test_parse_arguments_for_actions_consumes_action_arguments_after_action_name ) flexmock(module).should_receive('get_subactions_for_actions').and_return({}) action_parsers = {'action': flexmock(), 'other': flexmock()} - global_namespace = flexmock() + global_namespace = flexmock(config_paths=[]) global_parser = flexmock() global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) @@ -204,7 +204,7 @@ def test_parse_arguments_for_actions_consumes_action_arguments_with_alias(): 'other': flexmock(), '-o': flexmock(), } - global_namespace = flexmock() + global_namespace = flexmock(config_paths=[]) global_parser = flexmock() global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) flexmock(module).ACTION_ALIASES = {'action': ['-a'], 'other': ['-o']} @@ -232,7 +232,7 @@ def test_parse_arguments_for_actions_consumes_multiple_action_arguments(): 'action': flexmock(), 'other': flexmock(), } - global_namespace = flexmock() + global_namespace = flexmock(config_paths=[]) global_parser = flexmock() global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) @@ -263,7 +263,7 @@ def test_parse_arguments_for_actions_respects_command_line_action_ordering(): 'action': flexmock(), 'other': flexmock(), } - global_namespace = flexmock() + global_namespace = flexmock(config_paths=[]) global_parser = flexmock() global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) @@ -278,7 +278,7 @@ def test_parse_arguments_for_actions_respects_command_line_action_ordering(): def test_parse_arguments_for_actions_applies_default_action_parsers(): - global_namespace = flexmock() + global_namespace = flexmock(config_paths=[]) namespaces = { 'global': global_namespace, 'prune': flexmock(), @@ -327,7 +327,7 @@ def test_parse_arguments_for_actions_consumes_global_arguments(): 'action': flexmock(), 'other': flexmock(), } - global_namespace = flexmock() + global_namespace = flexmock(config_paths=[]) global_parser = flexmock() global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) @@ -353,7 +353,7 @@ def test_parse_arguments_for_actions_passes_through_unknown_arguments_before_act 'action': flexmock(), 'other': flexmock(), } - global_namespace = flexmock() + global_namespace = flexmock(config_paths=[]) global_parser = flexmock() global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) @@ -379,7 +379,7 @@ def test_parse_arguments_for_actions_passes_through_unknown_arguments_after_acti 'action': flexmock(), 'other': flexmock(), } - global_namespace = flexmock() + global_namespace = flexmock(config_paths=[]) global_parser = flexmock() global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) @@ -405,7 +405,7 @@ def test_parse_arguments_for_actions_with_borg_action_skips_other_action_parsers 'borg': flexmock(), 'list': flexmock(), } - global_namespace = flexmock() + global_namespace = flexmock(config_paths=[]) global_parser = flexmock() global_parser.should_receive('parse_known_args').and_return((global_namespace, ())) From 325b5612965cfff4f7cb8b9a3904f7eeddbd1270 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 24 Jun 2023 15:52:20 -0700 Subject: [PATCH 303/344] Switch from "init" to "rcreate" for creating repos in end-to-end tests. --- tests/end-to-end/test_borgmatic.py | 2 +- tests/end-to-end/test_database.py | 10 +++++----- tests/end-to-end/test_override.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/end-to-end/test_borgmatic.py b/tests/end-to-end/test_borgmatic.py index f2b81a81..93fd27a2 100644 --- a/tests/end-to-end/test_borgmatic.py +++ b/tests/end-to-end/test_borgmatic.py @@ -44,7 +44,7 @@ def test_borgmatic_command(): generate_configuration(config_path, repository_path) subprocess.check_call( - f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ') + f'borgmatic -v 2 --config {config_path} rcreate --encryption repokey'.split(' ') ) # Run borgmatic to generate a backup archive, and then list it to make sure it exists. diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index fd565b21..0a10339e 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -202,7 +202,7 @@ def test_database_dump_and_restore(): ) subprocess.check_call( - ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] + ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey'] ) # Run borgmatic to generate a backup archive including a database dump. @@ -242,7 +242,7 @@ def test_database_dump_and_restore_with_restore_cli_arguments(): ) subprocess.check_call( - ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] + ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey'] ) # Run borgmatic to generate a backup archive including a database dump. @@ -299,7 +299,7 @@ def test_database_dump_and_restore_with_restore_configuration_options(): ) subprocess.check_call( - ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] + ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey'] ) # Run borgmatic to generate a backup archive including a database dump. @@ -344,7 +344,7 @@ def test_database_dump_and_restore_with_directory_format(): ) subprocess.check_call( - ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] + ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey'] ) # Run borgmatic to generate a backup archive including a database dump. @@ -374,7 +374,7 @@ def test_database_dump_with_error_causes_borgmatic_to_exit(): ) subprocess.check_call( - ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey'] + ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey'] ) # Run borgmatic with a config override such that the database dump fails. diff --git a/tests/end-to-end/test_override.py b/tests/end-to-end/test_override.py index e91e28bf..7debc6e2 100644 --- a/tests/end-to-end/test_override.py +++ b/tests/end-to-end/test_override.py @@ -39,7 +39,7 @@ def test_override_get_normalized(): generate_configuration(config_path, repository_path) subprocess.check_call( - f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ') + f'borgmatic -v 2 --config {config_path} rcreate --encryption repokey'.split(' ') ) # Run borgmatic with an override structured for an outdated config file format. If From 37a0a0c42178692a65560e3391a559ad2d5fd800 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 24 Jun 2023 22:23:01 -0700 Subject: [PATCH 304/344] Bump version for release. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index f9d27518..586fe1de 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.7.15.dev0 +1.7.15 * #326: Add configuration options and command-line flags for backing up a database from one location while restoring it somewhere else. * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors. diff --git a/setup.py b/setup.py index 4fa049e0..65db6fc2 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.15.dev0' +VERSION = '1.7.15' setup( From b9a11e860d2a457ad85aaf3d6f3b224af94112ed Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 25 Jun 2023 15:36:25 -0700 Subject: [PATCH 305/344] Remove legacy configuration parsing code, no longer needed with upgrade-borgmatic-config gone (#529). --- borgmatic/commands/borgmatic.py | 2 - borgmatic/config/legacy.py | 146 ---------------- tests/integration/config/test_legacy.py | 18 -- tests/unit/config/test_legacy.py | 210 ------------------------ 4 files changed, 376 deletions(-) delete mode 100644 borgmatic/config/legacy.py delete mode 100644 tests/integration/config/test_legacy.py delete mode 100644 tests/unit/config/test_legacy.py diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 7d31ddee..6aaad9d1 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -46,8 +46,6 @@ from borgmatic.verbosity import verbosity_to_log_level logger = logging.getLogger(__name__) -LEGACY_CONFIG_PATH = '/etc/borgmatic/config' - def run_configuration(config_filename, config, arguments): ''' diff --git a/borgmatic/config/legacy.py b/borgmatic/config/legacy.py deleted file mode 100644 index ec1e50a1..00000000 --- a/borgmatic/config/legacy.py +++ /dev/null @@ -1,146 +0,0 @@ -from collections import OrderedDict, namedtuple -from configparser import RawConfigParser - -Section_format = namedtuple('Section_format', ('name', 'options')) -Config_option = namedtuple('Config_option', ('name', 'value_type', 'required')) - - -def option(name, value_type=str, required=True): - ''' - Given a config file option name, an expected type for its value, and whether it's required, - return a Config_option capturing that information. - ''' - return Config_option(name, value_type, required) - - -CONFIG_FORMAT = ( - Section_format( - 'location', - ( - option('source_directories'), - option('one_file_system', value_type=bool, required=False), - option('remote_path', required=False), - option('repository'), - ), - ), - Section_format( - 'storage', - ( - option('encryption_passphrase', required=False), - option('compression', required=False), - option('umask', required=False), - ), - ), - Section_format( - 'retention', - ( - option('keep_within', required=False), - option('keep_hourly', int, required=False), - option('keep_daily', int, required=False), - option('keep_weekly', int, required=False), - option('keep_monthly', int, required=False), - option('keep_yearly', int, required=False), - option('prefix', required=False), - ), - ), - Section_format( - 'consistency', (option('checks', required=False), option('check_last', required=False)) - ), -) - - -def validate_configuration_format(parser, config_format): - ''' - Given an open RawConfigParser and an expected config file format, validate that the parsed - configuration file has the expected sections, that any required options are present in those - sections, and that there aren't any unexpected options. - - A section is required if any of its contained options are required. - - Raise ValueError if anything is awry. - ''' - section_names = set(parser.sections()) - required_section_names = tuple( - section.name - for section in config_format - if any(option.required for option in section.options) - ) - - unknown_section_names = section_names - set( - section_format.name for section_format in config_format - ) - if unknown_section_names: - raise ValueError(f"Unknown config sections found: {', '.join(unknown_section_names)}") - - missing_section_names = set(required_section_names) - section_names - if missing_section_names: - raise ValueError(f"Missing config sections: {', '.join(missing_section_names)}") - - for section_format in config_format: - if section_format.name not in section_names: - continue - - option_names = parser.options(section_format.name) - expected_options = section_format.options - - unexpected_option_names = set(option_names) - set( - option.name for option in expected_options - ) - - if unexpected_option_names: - raise ValueError( - f"Unexpected options found in config section {section_format.name}: {', '.join(sorted(unexpected_option_names))}", - ) - - missing_option_names = tuple( - option.name - for option in expected_options - if option.required - if option.name not in option_names - ) - - if missing_option_names: - raise ValueError( - f"Required options missing from config section {section_format.name}: {', '.join(missing_option_names)}", - ) - - -def parse_section_options(parser, section_format): - ''' - Given an open RawConfigParser and an expected section format, return the option values from that - section as a dict mapping from option name to value. Omit those options that are not present in - the parsed options. - - Raise ValueError if any option values cannot be coerced to the expected Python data type. - ''' - type_getter = {str: parser.get, int: parser.getint, bool: parser.getboolean} - - return OrderedDict( - (option.name, type_getter[option.value_type](section_format.name, option.name)) - for option in section_format.options - if parser.has_option(section_format.name, option.name) - ) - - -def parse_configuration(config_filename, config_format): - ''' - Given a config filename and an expected config file format, return the parsed configuration - as a namedtuple with one attribute for each parsed section. - - Raise IOError if the file cannot be read, or ValueError if the format is not as expected. - ''' - parser = RawConfigParser() - if not parser.read(config_filename): - raise ValueError(f'Configuration file cannot be opened: {config_filename}') - - validate_configuration_format(parser, config_format) - - # Describes a parsed configuration, where each attribute is the name of a configuration file - # section and each value is a dict of that section's parsed options. - Parsed_config = namedtuple( - 'Parsed_config', (section_format.name for section_format in config_format) - ) - - return Parsed_config( - *(parse_section_options(parser, section_format) for section_format in config_format) - ) diff --git a/tests/integration/config/test_legacy.py b/tests/integration/config/test_legacy.py deleted file mode 100644 index c73e7eec..00000000 --- a/tests/integration/config/test_legacy.py +++ /dev/null @@ -1,18 +0,0 @@ -import string -from collections import OrderedDict -from io import StringIO - -from borgmatic.config import legacy as module - - -def test_parse_section_options_with_punctuation_should_return_section_options(): - parser = module.RawConfigParser() - parser.read_file(StringIO(f'[section]\nfoo: {string.punctuation}\n')) - - section_format = module.Section_format( - 'section', (module.Config_option('foo', str, required=True),) - ) - - config = module.parse_section_options(parser, section_format) - - assert config == OrderedDict((('foo', string.punctuation),)) diff --git a/tests/unit/config/test_legacy.py b/tests/unit/config/test_legacy.py deleted file mode 100644 index 230aa747..00000000 --- a/tests/unit/config/test_legacy.py +++ /dev/null @@ -1,210 +0,0 @@ -from collections import OrderedDict - -import pytest -from flexmock import flexmock - -from borgmatic.config import legacy as module - - -def test_option_should_create_config_option(): - option = module.option('name', bool, required=False) - - assert option == module.Config_option('name', bool, False) - - -def test_option_should_create_config_option_with_defaults(): - option = module.option('name') - - assert option == module.Config_option('name', str, True) - - -def test_validate_configuration_format_with_valid_config_should_not_raise(): - parser = flexmock() - parser.should_receive('sections').and_return(('section', 'other')) - parser.should_receive('options').with_args('section').and_return(('stuff',)) - parser.should_receive('options').with_args('other').and_return(('such',)) - config_format = ( - module.Section_format( - 'section', options=(module.Config_option('stuff', str, required=True),) - ), - module.Section_format('other', options=(module.Config_option('such', str, required=True),)), - ) - - module.validate_configuration_format(parser, config_format) - - -def test_validate_configuration_format_with_missing_required_section_should_raise(): - parser = flexmock() - parser.should_receive('sections').and_return(('section',)) - config_format = ( - module.Section_format( - 'section', options=(module.Config_option('stuff', str, required=True),) - ), - # At least one option in this section is required, so the section is required. - module.Section_format( - 'missing', - options=( - module.Config_option('such', str, required=False), - module.Config_option('things', str, required=True), - ), - ), - ) - - with pytest.raises(ValueError): - module.validate_configuration_format(parser, config_format) - - -def test_validate_configuration_format_with_missing_optional_section_should_not_raise(): - parser = flexmock() - parser.should_receive('sections').and_return(('section',)) - parser.should_receive('options').with_args('section').and_return(('stuff',)) - config_format = ( - module.Section_format( - 'section', options=(module.Config_option('stuff', str, required=True),) - ), - # No options in the section are required, so the section is optional. - module.Section_format( - 'missing', - options=( - module.Config_option('such', str, required=False), - module.Config_option('things', str, required=False), - ), - ), - ) - - module.validate_configuration_format(parser, config_format) - - -def test_validate_configuration_format_with_unknown_section_should_raise(): - parser = flexmock() - parser.should_receive('sections').and_return(('section', 'extra')) - config_format = (module.Section_format('section', options=()),) - - with pytest.raises(ValueError): - module.validate_configuration_format(parser, config_format) - - -def test_validate_configuration_format_with_missing_required_option_should_raise(): - parser = flexmock() - parser.should_receive('sections').and_return(('section',)) - parser.should_receive('options').with_args('section').and_return(('option',)) - config_format = ( - module.Section_format( - 'section', - options=( - module.Config_option('option', str, required=True), - module.Config_option('missing', str, required=True), - ), - ), - ) - - with pytest.raises(ValueError): - module.validate_configuration_format(parser, config_format) - - -def test_validate_configuration_format_with_missing_optional_option_should_not_raise(): - parser = flexmock() - parser.should_receive('sections').and_return(('section',)) - parser.should_receive('options').with_args('section').and_return(('option',)) - config_format = ( - module.Section_format( - 'section', - options=( - module.Config_option('option', str, required=True), - module.Config_option('missing', str, required=False), - ), - ), - ) - - module.validate_configuration_format(parser, config_format) - - -def test_validate_configuration_format_with_extra_option_should_raise(): - parser = flexmock() - parser.should_receive('sections').and_return(('section',)) - parser.should_receive('options').with_args('section').and_return(('option', 'extra')) - config_format = ( - module.Section_format( - 'section', options=(module.Config_option('option', str, required=True),) - ), - ) - - with pytest.raises(ValueError): - module.validate_configuration_format(parser, config_format) - - -def test_parse_section_options_should_return_section_options(): - parser = flexmock() - parser.should_receive('get').with_args('section', 'foo').and_return('value') - parser.should_receive('getint').with_args('section', 'bar').and_return(1) - parser.should_receive('getboolean').never() - parser.should_receive('has_option').with_args('section', 'foo').and_return(True) - parser.should_receive('has_option').with_args('section', 'bar').and_return(True) - - section_format = module.Section_format( - 'section', - ( - module.Config_option('foo', str, required=True), - module.Config_option('bar', int, required=True), - ), - ) - - config = module.parse_section_options(parser, section_format) - - assert config == OrderedDict((('foo', 'value'), ('bar', 1))) - - -def test_parse_section_options_for_missing_section_should_return_empty_dict(): - parser = flexmock() - parser.should_receive('get').never() - parser.should_receive('getint').never() - parser.should_receive('getboolean').never() - parser.should_receive('has_option').with_args('section', 'foo').and_return(False) - parser.should_receive('has_option').with_args('section', 'bar').and_return(False) - - section_format = module.Section_format( - 'section', - ( - module.Config_option('foo', str, required=False), - module.Config_option('bar', int, required=False), - ), - ) - - config = module.parse_section_options(parser, section_format) - - assert config == OrderedDict() - - -def insert_mock_parser(): - parser = flexmock() - parser.should_receive('read').and_return([flexmock()]) - module.RawConfigParser = lambda: parser - - return parser - - -def test_parse_configuration_should_return_section_configs(): - parser = insert_mock_parser() - config_format = (flexmock(name='items'), flexmock(name='things')) - mock_module = flexmock(module) - mock_module.should_receive('validate_configuration_format').with_args( - parser, config_format - ).once() - mock_section_configs = (flexmock(), flexmock()) - - for section_format, section_config in zip(config_format, mock_section_configs): - mock_module.should_receive('parse_section_options').with_args( - parser, section_format - ).and_return(section_config).once() - - parsed_config = module.parse_configuration('filename', config_format) - - assert parsed_config == type(parsed_config)(*mock_section_configs) - - -def test_parse_configuration_with_file_open_error_should_raise(): - parser = insert_mock_parser() - parser.should_receive('read').and_return([]) - - with pytest.raises(ValueError): - module.parse_configuration('filename', config_format=flexmock()) From c3004c609060012a91ce14cc0742b7cc620dfe17 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 25 Jun 2023 22:49:36 -0700 Subject: [PATCH 306/344] Some brief documentation on running only checks (#364). --- docs/how-to/backup-your-databases.md | 5 +++-- docs/how-to/deal-with-very-large-backups.md | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index fea3f350..bf2c7b68 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -189,8 +189,9 @@ system. would like to backup databases only and not source directories, you can omit `source_directories` entirely. -In older versions of borgmatic, instead specify an empty `source_directories` -value, as it is a mandatory option prior to version 1.7.1: +Prior to version 1.7.1 In older +versions of borgmatic, instead specify an empty `source_directories` value, as +it is a mandatory option there: ```yaml location: diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index 5beb9f24..d6142619 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -137,6 +137,23 @@ If you want to temporarily ignore your configured frequencies, you can invoke `borgmatic check --force` to run checks unconditionally. +### Running only checks + +New in version 1.7.1 If you +would like to only run consistency checks without creating backups (for +instance with the `check` action on the command-line), you can omit +the `source_directories` option entirely. + +Prior to version 1.7.1 In older +versions of borgmatic, instead specify an empty `source_directories` value, as +it is a mandatory option there: + +```yaml +location: + source_directories: [] +``` + + ### Disabling checks If that's still too slow, you can disable consistency checks entirely, From b242078f546b8c52f7f032513254c1c220891909 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 26 Jun 2023 09:30:46 -0700 Subject: [PATCH 307/344] Fix an error when running "borg key export" through borgmatic (#719). --- NEWS | 3 +++ borgmatic/commands/arguments.py | 12 +++++++++--- setup.py | 2 +- tests/integration/commands/test_arguments.py | 11 +++++++++++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 586fe1de..41ee5fef 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.7.16.dev0 + * #719: Fix an error when running "borg key export" through borgmatic. + 1.7.15 * #326: Add configuration options and command-line flags for backing up a database from one location while restoring it somewhere else. diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6b857f2c..7e054f5a 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -216,15 +216,21 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse arguments['global'], remaining = global_parser.parse_known_args(unparsed_arguments) remaining_action_arguments.append(remaining) - # Prevent action names that follow "--config" paths from being considered as additional paths. + # Prevent action names and arguments that follow "--config" paths from being considered as + # additional paths. for argument_name in arguments.keys(): if argument_name == 'global': continue for action_name in [argument_name] + ACTION_ALIASES.get(argument_name, []): - if action_name in arguments['global'].config_paths: - arguments['global'].config_paths.remove(action_name) + try: + action_name_index = arguments['global'].config_paths.index(action_name) + arguments['global'].config_paths = arguments['global'].config_paths[ + :action_name_index + ] break + except ValueError: + pass return ( arguments, diff --git a/setup.py b/setup.py index 65db6fc2..37ff017a 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.15' +VERSION = '1.7.16.dev0' setup( diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index f992f9aa..27dc3234 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -52,6 +52,17 @@ def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): assert arguments['rcreate'].encryption_mode == 'repokey' +def test_parse_arguments_with_action_and_positional_arguments_after_config_path_omits_action_and_arguments(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + arguments = module.parse_arguments('--config', 'myconfig', 'borg', 'key', 'export') + + global_arguments = arguments['global'] + assert global_arguments.config_paths == ['myconfig'] + assert 'borg' in arguments + assert arguments['borg'].options == ['key', 'export'] + + def test_parse_arguments_with_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) From bb6004fc4f5671f94f6f92a704ebfc44d533a6cf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 26 Jun 2023 14:35:07 -0700 Subject: [PATCH 308/344] Revamp "borg" action to support REPOSITORY and ARCHIVE env vars instead of implicitly injecting repository/archive into the Borg command (#575). --- NEWS | 6 +- borgmatic/borg/borg.py | 25 ++-- docs/how-to/run-arbitrary-borg-commands.md | 102 ++++++++++----- setup.py | 2 +- tests/unit/borg/test_borg.py | 142 ++++++++------------- 5 files changed, 140 insertions(+), 137 deletions(-) diff --git a/NEWS b/NEWS index 41ee5fef..978d04fd 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,8 @@ -1.7.16.dev0 +1.8.0.dev0 + * #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting + repository/archive into the resulting Borg command-line, make repository and archive environment + variables available for explicit use in your commands. See the documentation for more + information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/ * #719: Fix an error when running "borg key export" through borgmatic. 1.7.15 diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index f815dfd0..c5adf892 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -10,7 +10,6 @@ logger = logging.getLogger(__name__) REPOSITORYLESS_BORG_COMMANDS = {'serve', None} BORG_SUBCOMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'} -BORG_SUBCOMMANDS_WITHOUT_REPOSITORY = (('debug', 'info'), ('debug', 'convert-profile'), ()) def run_arbitrary_borg( @@ -25,7 +24,8 @@ def run_arbitrary_borg( ''' Given a local or remote repository path, a storage config dict, the local Borg version, a sequence of arbitrary command-line Borg options, and an optional archive name, run an arbitrary - Borg command on the given repository/archive. + Borg command, passing in $REPOSITORY and $ARCHIVE environment variables for optional use in the + commmand. ''' borgmatic.logger.add_custom_log_levels() lock_wait = storage_config.get('lock_wait', None) @@ -46,29 +46,26 @@ def run_arbitrary_borg( borg_command = () command_options = () - if borg_command in BORG_SUBCOMMANDS_WITHOUT_REPOSITORY: - repository_archive_flags = () - elif archive: - repository_archive_flags = flags.make_repository_archive_flags( - repository_path, archive, local_borg_version - ) - else: - repository_archive_flags = flags.make_repository_flags(repository_path, local_borg_version) - full_command = ( (local_path,) + borg_command - + repository_archive_flags - + command_options + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + flags.make_flags('lock-wait', lock_wait) + + command_options ) return execute_command( full_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), + shell=True, + extra_environment=dict( + (environment.make_environment(storage_config) or {}), + **{ + 'REPOSITORY': repository_path, + 'ARCHIVE': archive if archive else '', + }, + ), ) diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index ea265eaa..e5720381 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -7,7 +7,7 @@ eleventyNavigation: --- ## Running Borg with borgmatic -Borg has several commands (and options) that borgmatic does not currently +Borg has several commands and options that borgmatic does not currently support. Sometimes though, as a borgmatic user, you may find yourself wanting to take advantage of these off-the-beaten-path Borg features. You could of course drop down to running Borg directly. But then you'd give up all the @@ -17,11 +17,11 @@ request](https://torsion.org/borgmatic/#contributing) to add the feature. But what if you need it *now*? That's where borgmatic's support for running "arbitrary" Borg commands comes -in. Running Borg commands with borgmatic takes advantage of the following, all -based on your borgmatic configuration files or command-line arguments: +in. Running these Borg commands with borgmatic can take advantage of the +following, all based on your borgmatic configuration files or command-line +arguments: - * configured repositories (automatically runs your Borg command once for each - one) + * configured repositories, running your Borg command once for each one * local and remote Borg binary paths * SSH settings and Borg environment variables * lock wait settings @@ -33,37 +33,78 @@ based on your borgmatic configuration files or command-line arguments: New in version 1.5.15 The way you run Borg with borgmatic is via the `borg` action. Here's a simple example: +```bash +borgmatic borg break-lock '$REPOSITORY' +``` + +This runs Borg's `break-lock` command once on each configured borgmatic +repository, passing the repository path in as an environment variable named +`REPOSITORY`. The single quotes are necessary in order to pass in a literal +`$REPOSITORY` string instead of trying to resolve it from borgmatic's shell +where it's not yet set. + +Prior to version 1.8.0borgmatic +provided the repository name implicitly, attempting to inject it into your +Borg arguments in the right place (which didn't always work). So your +command-line in these older versions looked more like: + ```bash borgmatic borg break-lock ``` -(No `borg` action in borgmatic? Time to upgrade!) - -This runs Borg's `break-lock` command once on each configured borgmatic -repository. Notice how the repository isn't present in the specified Borg -options, as that part is provided by borgmatic. - -You can also specify Borg options for relevant commands: +You can also specify Borg options for relevant commands. In borgmatic 1.8.0+, +that looks like: ```bash -borgmatic borg rlist --short +borgmatic borg rlist --short '$REPOSITORY' ``` This runs Borg's `rlist` command once on each configured borgmatic repository. -(The native `borgmatic rlist` action should be preferred for most use.) +However, the native `borgmatic rlist` action should be preferred for most uses. What if you only want to run Borg on a single configured borgmatic repository when you've got several configured? Not a problem. The `--repository` argument lets you specify the repository to use, either by its path or its label: ```bash -borgmatic borg --repository repo.borg break-lock +borgmatic borg --repository repo.borg break-lock '$REPOSITORY' ``` -And what about a single archive? +### Specifying an archive + +For borg commands that expect an archive name, you have a few approaches. +Here's one: ```bash -borgmatic borg --archive your-archive-name rlist +borgmatic borg --archive latest list '$REPOSITORY::$ARCHIVE' +``` + +Or if you don't need borgmatic to resolve an archive name like `latest`, you +can just do: + +```bash +borgmatic borg list '$REPOSITORY::your-actual-archive-name' +``` + +Prior to version 1.8.0borgmatic +provided the archive name implicitly along with the repository, attempting to +inject it into your Borg arguments in the right place (which didn't always +work). So your command-line in these older versions of borgmatic looked more +like: + +```bash +borgmatic borg --archive latest list +``` + +With Borg version 2.x Either of +these will list an archive: + +```bash +borgmatic borg --archive latest list --repo '$REPOSITORY' '$ARCHIVE' +``` + +```bash +borgmatic borg list --repo '$REPOSITORY' your-actual-archive-name ``` ### Limitations @@ -71,14 +112,10 @@ borgmatic borg --archive your-archive-name rlist borgmatic's `borg` action is not without limitations: * The Borg command you want to run (`create`, `list`, etc.) *must* come first - after the `borg` action. If you have any other Borg options to specify, - provide them after. For instance, `borgmatic borg list --progress` will work, - but `borgmatic borg --progress list` will not. - * borgmatic supplies the repository/archive name to Borg for you (based on - your borgmatic configuration or the `borgmatic borg --repository`/`--archive` - arguments), so do not specify the repository/archive otherwise. - * The `borg` action will not currently work for any Borg commands like `borg - serve` that do not accept a repository/archive name. + after the `borg` action (and any borgmatic-specific arguments). If you have + other Borg options to specify, provide them after. For instance, + `borgmatic borg list --progress ...` will work, but + `borgmatic borg --progress list ...` will not. * Do not specify any global borgmatic arguments to the right of the `borg` action. (They will be passed to Borg instead of borgmatic.) If you have global borgmatic arguments, specify them *before* the `borg` action. @@ -88,10 +125,17 @@ borgmatic's `borg` action is not without limitations: borgmatic action. In this case, only the Borg command is run. * Unlike normal borgmatic actions that support JSON, the `borg` action will not disable certain borgmatic logs to avoid interfering with JSON output. - * Unlike other borgmatic actions, the `borg` action captures (and logs) all - output, so interactive prompts and flags like `--progress` will not work as - expected. New in version - 1.7.13 borgmatic now runs the `borg` action without capturing output, + * Prior to version 1.8.0 + borgmatic implicitly supplied the repository/archive name to Borg for you + (based on your borgmatic configuration or the + `borgmatic borg --repository`/`--archive` arguments)—which meant you couldn't + specify the repository/archive directly in the Borg command. Also, in these + older versions of borgmatic, the `borg` action didn't work for any Borg + commands like `borg serve` that do not accept a repository/archive name. + * Prior to version 1.7.13 Unlike + other borgmatic actions, the `borg` action captured (and logged) all output, + so interactive prompts and flags like `--progress` dit not work as expected. + In new versions, borgmatic runs the `borg` action without capturing output, so interactive prompts work. In general, this `borgmatic borg` feature should be considered an escape diff --git a/setup.py b/setup.py index 37ff017a..d6a01fd6 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.16.dev0' +VERSION = '1.8.0.dev0' setup( diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 5ae013f8..f03bb89f 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -10,35 +10,35 @@ from ..test_verbosity import insert_logging_mock def test_run_arbitrary_borg_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo'), + ('borg', 'break-lock', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], ) def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo', '--info'), + ('borg', 'break-lock', '--info', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.INFO) @@ -46,21 +46,21 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], ) def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo', '--debug', '--show-rc'), + ('borg', 'break-lock', '--debug', '--show-rc', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.DEBUG) @@ -68,7 +68,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], ) @@ -76,46 +76,44 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo', '--lock-wait', '5'), + ('borg', 'break-lock', '--lock-wait', '5', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], ) def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( - ('repo::archive',) - ) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo::archive'), + ('borg', 'break-lock', '$REPOSITORY::$ARCHIVE'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': 'archive'}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY::$ARCHIVE'], archive='archive', ) @@ -123,21 +121,21 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg1', 'break-lock', 'repo'), + ('borg1', 'break-lock', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg1', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], local_path='borg1', ) @@ -145,23 +143,23 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return( ('--remote-path', 'borg1') ).and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo', '--remote-path', 'borg1'), + ('borg', 'break-lock', '--remote-path', 'borg1', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], remote_path='borg1', ) @@ -169,56 +167,56 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags() def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', 'repo', '--progress'), + ('borg', 'list', '--progress', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['list', '--progress'], + options=['list', '--progress', '$REPOSITORY'], ) def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo'), + ('borg', 'break-lock', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['--', 'break-lock'], + options=['--', 'break-lock', '$REPOSITORY'], ) def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').never() flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg',), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( @@ -229,85 +227,45 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): ) -def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository(): +def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'key', 'export', 'repo'), + ('borg', 'key', 'export', '--info', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) + insert_logging_mock(logging.INFO) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['key', 'export'], + options=['key', 'export', '$REPOSITORY'], ) -def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository(): +def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'debug', 'dump-manifest', 'repo', 'path'), + ('borg', 'debug', 'dump-manifest', '--info', '$REPOSITORY', 'path'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) + insert_logging_mock(logging.INFO) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['debug', 'dump-manifest', 'path'], - ) - - -def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repository(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').never() - flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'debug', 'info'), - output_file=module.borgmatic.execute.DO_NOT_CAPTURE, - borg_local_path='borg', - extra_environment=None, - ) - - module.run_arbitrary_borg( - repository_path='repo', - storage_config={}, - local_borg_version='1.2.3', - options=['debug', 'info'], - ) - - -def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_borg_repository(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').never() - flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'debug', 'convert-profile', 'in', 'out'), - output_file=module.borgmatic.execute.DO_NOT_CAPTURE, - borg_local_path='borg', - extra_environment=None, - ) - - module.run_arbitrary_borg( - repository_path='repo', - storage_config={}, - local_borg_version='1.2.3', - options=['debug', 'convert-profile', 'in', 'out'], + options=['debug', 'dump-manifest', '$REPOSITORY', 'path'], ) From 44f9ab95f9c522fc7484a76fcded5ee261220ced Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 26 Jun 2023 14:37:23 -0700 Subject: [PATCH 309/344] Fix typos (#575). --- borgmatic/borg/borg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index c5adf892..107f140a 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -24,8 +24,8 @@ def run_arbitrary_borg( ''' Given a local or remote repository path, a storage config dict, the local Borg version, a sequence of arbitrary command-line Borg options, and an optional archive name, run an arbitrary - Borg command, passing in $REPOSITORY and $ARCHIVE environment variables for optional use in the - commmand. + Borg command, passing in REPOSITORY and ARCHIVE environment variables for optional use in the + command. ''' borgmatic.logger.add_custom_log_levels() lock_wait = storage_config.get('lock_wait', None) From f60e97d5bfbbbcc770fc928c1db817c65a2af797 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 26 Jun 2023 16:46:09 -0700 Subject: [PATCH 310/344] When merging two configuration files, error gracefully if the two files do not adhere to the same format. --- NEWS | 2 ++ borgmatic/config/load.py | 7 ++++ tests/integration/config/test_load.py | 48 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/NEWS b/NEWS index 978d04fd..a63be326 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,8 @@ variables available for explicit use in your commands. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/ * #719: Fix an error when running "borg key export" through borgmatic. + * When merging two configuration files, error gracefully if the two files do not adhere to the same + format. 1.7.15 * #326: Add configuration options and command-line flags for backing up a database from one diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index f6290de8..f5d071c9 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -225,6 +225,8 @@ def deep_merge_nodes(nodes): The purpose of deep merging like this is to support, for instance, merging one borgmatic configuration file into another for reuse, such that a configuration section ("retention", etc.) does not completely replace the corresponding section in a merged file. + + Raise ValueError if a merge is implied using two incompatible types. ''' # Map from original node key/value to the replacement merged node. DELETED_NODE as a replacement # node indications deletion. @@ -239,6 +241,11 @@ def deep_merge_nodes(nodes): # If the keys match and the values are different, we need to merge these two A and B nodes. if a_key.tag == b_key.tag and a_key.value == b_key.value and a_value != b_value: + if not type(a_value) is type(b_value): + raise ValueError( + f'Incompatible types found when trying to merge "{a_key.value}:" values across configuration files: {type(a_value).id} and {type(b_value).id}' + ) + # Since we're merging into the B node, consider the A node a duplicate and remove it. replaced_nodes[(a_key, a_value)] = DELETED_NODE diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index 028a6523..81b5deee 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -702,6 +702,54 @@ def test_deep_merge_nodes_appends_colliding_sequence_values(): assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 2', 'echo 3', 'echo 4'] +def test_deep_merge_nodes_errors_on_colliding_values_of_different_types(): + node_values = [ + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo oopsie daisy' + ), + ), + ], + ), + ), + ( + module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'), + module.ruamel.yaml.nodes.MappingNode( + tag='tag:yaml.org,2002:map', + value=[ + ( + module.ruamel.yaml.nodes.ScalarNode( + tag='tag:yaml.org,2002:str', value='before_backup' + ), + module.ruamel.yaml.nodes.SequenceNode( + tag='tag:yaml.org,2002:seq', + value=[ + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 3' + ), + module.ruamel.yaml.ScalarNode( + tag='tag:yaml.org,2002:str', value='echo 4' + ), + ], + ), + ), + ], + ), + ), + ] + + with pytest.raises(ValueError): + module.deep_merge_nodes(node_values) + + def test_deep_merge_nodes_only_keeps_mapping_values_tagged_with_retain(): node_values = [ ( From e2c95327fb2a3a0bd7b9a6bcfb042b1832a5d5de Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 28 Jun 2023 09:15:11 -0700 Subject: [PATCH 311/344] Fix an error when dumping a MySQL database and the "exclude_nodump" option is set (#720). --- NEWS | 1 + borgmatic/borg/create.py | 7 +++++-- tests/unit/borg/test_create.py | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index a63be326..341983b3 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ variables available for explicit use in your commands. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/ * #719: Fix an error when running "borg key export" through borgmatic. + * #720: Fix an error when dumping a MySQL database and the "exclude_nodump" option is set. * When merging two configuration files, error gracefully if the two files do not adhere to the same format. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 618376a5..1ec54fdd 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -280,14 +280,17 @@ def collect_special_file_paths( create_command, local_path, working_directory, borg_environment, skip_directories ): ''' - Given a Borg create command as a tuple, a local Borg path, a working directory, and a dict of + Given a Borg create command as a tuple, a local Borg path, a working directory, a dict of environment variables to pass to Borg, and a sequence of parent directories to skip, collect the paths for any special files (character devices, block devices, and named pipes / FIFOs) that Borg would encounter during a create. These are all paths that could cause Borg to hang if its --read-special flag is used. ''' + # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open + # files including any named pipe we've created. paths_output = execute_command_and_capture_output( - create_command + ('--dry-run', '--list'), + tuple(argument for argument in create_command if argument != '--exclude-nodump') + + ('--dry-run', '--list'), capture_stderr=True, working_directory=working_directory, extra_environment=borg_environment, diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index c798c446..33e95607 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -449,6 +449,25 @@ def test_collect_special_file_paths_excludes_non_special_files(): ) == ('/foo', '/baz') +def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command(): + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ('borg', 'create', '--dry-run', '--list'), + capture_stderr=True, + working_directory=None, + extra_environment=None, + ).and_return('Processing files ...\n- /foo\n+ /bar\n- /baz').once() + flexmock(module).should_receive('special_file').and_return(True) + flexmock(module).should_receive('any_parent_directories').and_return(False) + + module.collect_special_file_paths( + ('borg', 'create', '--exclude-nodump'), + local_path=None, + working_directory=None, + borg_environment=None, + skip_directories=flexmock(), + ) + + DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003 REPO_ARCHIVE_WITH_PATHS = (f'repo::{DEFAULT_ARCHIVE_NAME}', 'foo', 'bar') From 9cf27fa4badfd228d3cbc5bcbc37b3c379854bd1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 29 Jun 2023 10:03:36 -0700 Subject: [PATCH 312/344] Deprecated configuration options warning logging. --- NEWS | 5 + borgmatic/borg/list.py | 1 - borgmatic/commands/arguments.py | 16 --- borgmatic/config/normalize.py | 106 ++++++++++++++++++- tests/integration/commands/test_arguments.py | 11 -- tests/integration/config/test_validate.py | 18 ++-- tests/unit/config/test_normalize.py | 34 +++--- 7 files changed, 140 insertions(+), 51 deletions(-) diff --git a/NEWS b/NEWS index 341983b3..1af50d8f 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,11 @@ * #720: Fix an error when dumping a MySQL database and the "exclude_nodump" option is set. * When merging two configuration files, error gracefully if the two files do not adhere to the same format. + * BREAKING: Remove the deprecated (and silently ignored) "--successful" flag on the "list" action, + as newer versions of Borg list successful (non-checkpoint) archives by default. + * All deprecated configuration option values now generate warning logs. + * Remove the deprecated (and non-functional) "--excludes" flag in favor of excludes within + configuration. 1.7.15 * #326: Add configuration options and command-line flags for backing up a database from one diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 96a6a87f..5ec1480d 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -14,7 +14,6 @@ ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST = ('prefix', 'match_archives', 'sort_by', 'f MAKE_FLAGS_EXCLUDES = ( 'repository', 'archive', - 'successful', 'paths', 'find_paths', ) + ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 7e054f5a..588e31c9 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -274,11 +274,6 @@ def make_parsers(): default=config_paths, help=f"Configuration filenames or directories, defaults to: {' '.join(unexpanded_config_paths)}", ) - global_group.add_argument( - '--excludes', - dest='excludes_filename', - help='Deprecated in favor of exclude_patterns within configuration', - ) global_group.add_argument( '-n', '--dry-run', @@ -1098,12 +1093,6 @@ def make_parsers(): metavar='PATTERN', help='Only list archive names matching this pattern', ) - list_group.add_argument( - '--successful', - default=True, - action='store_true', - help='Deprecated; no effect. Newer versions of Borg shows successful (non-checkpoint) archives by default.', - ) list_group.add_argument( '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys' ) @@ -1279,11 +1268,6 @@ def parse_arguments(*unparsed_arguments): f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}" ) - if arguments['global'].excludes_filename: - raise ValueError( - 'The --excludes flag has been replaced with exclude_patterns in configuration.' - ) - if 'create' in arguments and arguments['create'].list_files and arguments['create'].progress: raise ValueError( 'With the create action, only one of --list (--files) and --progress flags can be used.' diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 147e4e4e..daadfeb4 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -12,52 +12,143 @@ def normalize(config_filename, config): location = config.get('location') or {} storage = config.get('storage') or {} consistency = config.get('consistency') or {} + retention = config.get('retention') or {} hooks = config.get('hooks') or {} # Upgrade exclude_if_present from a string to a list. exclude_if_present = location.get('exclude_if_present') if isinstance(exclude_if_present, str): + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The exclude_if_present option now expects a list value. String values for this option are deprecated and support will be removed from a future release.', + ) + ) + ) config['location']['exclude_if_present'] = [exclude_if_present] # Upgrade various monitoring hooks from a string to a dict. healthchecks = hooks.get('healthchecks') if isinstance(healthchecks, str): + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The healthchecks hook now expects a mapping value. String values for this option are deprecated and support will be removed from a future release.', + ) + ) + ) config['hooks']['healthchecks'] = {'ping_url': healthchecks} cronitor = hooks.get('cronitor') if isinstance(cronitor, str): + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.', + ) + ) + ) config['hooks']['cronitor'] = {'ping_url': cronitor} pagerduty = hooks.get('pagerduty') if isinstance(pagerduty, str): + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.', + ) + ) + ) config['hooks']['pagerduty'] = {'integration_key': pagerduty} cronhub = hooks.get('cronhub') if isinstance(cronhub, str): + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.', + ) + ) + ) config['hooks']['cronhub'] = {'ping_url': cronhub} # Upgrade consistency checks from a list of strings to a list of dicts. checks = consistency.get('checks') if isinstance(checks, list) and len(checks) and isinstance(checks[0], str): + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The checks option now expects a list of key/value pairs. Lists of strings for this option are deprecated and support will be removed from a future release.', + ) + ) + ) config['consistency']['checks'] = [{'name': check_type} for check_type in checks] # Rename various configuration options. numeric_owner = location.pop('numeric_owner', None) if numeric_owner is not None: + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The numeric_owner option has been renamed to numeric_ids. numeric_owner is deprecated and support will be removed from a future release.', + ) + ) + ) config['location']['numeric_ids'] = numeric_owner bsd_flags = location.pop('bsd_flags', None) if bsd_flags is not None: + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The bsd_flags option has been renamed to flags. bsd_flags is deprecated and support will be removed from a future release.', + ) + ) + ) config['location']['flags'] = bsd_flags remote_rate_limit = storage.pop('remote_rate_limit', None) if remote_rate_limit is not None: + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The remote_rate_limit option has been renamed to upload_rate_limit. remote_rate_limit is deprecated and support will be removed from a future release.', + ) + ) + ) config['storage']['upload_rate_limit'] = remote_rate_limit # Upgrade remote repositories to ssh:// syntax, required in Borg 2. repositories = location.get('repositories') if repositories: if isinstance(repositories[0], str): + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The repositories option now expects a list of key/value pairs. Lists of strings for this option are deprecated and support will be removed from a future release.', + ) + ) + ) config['location']['repositories'] = [ {'path': repository} for repository in repositories ] @@ -71,7 +162,7 @@ def normalize(config_filename, config): dict( levelno=logging.WARNING, levelname='WARNING', - msg=f'{config_filename}: Repository paths containing "~" are deprecated in borgmatic and no longer work in Borg 2.x+.', + msg=f'{config_filename}: Repository paths containing "~" are deprecated in borgmatic and support will be removed from a future release.', ) ) ) @@ -95,7 +186,7 @@ def normalize(config_filename, config): dict( levelno=logging.WARNING, levelname='WARNING', - msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository_path}" as "{rewritten_repository_path}"', + msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated and support will be removed from a future release. Interpreting "{repository_path}" as "{rewritten_repository_path}"', ) ) ) @@ -108,4 +199,15 @@ def normalize(config_filename, config): else: config['location']['repositories'].append(repository_dict) + if consistency.get('prefix') or retention.get('prefix'): + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The prefix option is deprecated and support will be removed from a future release. Use archive_name_format or match_archives instead.', + ) + ) + ) + return logs diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 27dc3234..db8db636 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -12,7 +12,6 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths - assert global_arguments.excludes_filename is None assert global_arguments.verbosity == 0 assert global_arguments.syslog_verbosity == 0 assert global_arguments.log_file_verbosity == 0 @@ -71,7 +70,6 @@ def test_parse_arguments_with_verbosity_overrides_default(): global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths - assert global_arguments.excludes_filename is None assert global_arguments.verbosity == 1 assert global_arguments.syslog_verbosity == 0 assert global_arguments.log_file_verbosity == 0 @@ -85,7 +83,6 @@ def test_parse_arguments_with_syslog_verbosity_overrides_default(): global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths - assert global_arguments.excludes_filename is None assert global_arguments.verbosity == 0 assert global_arguments.syslog_verbosity == 2 @@ -98,7 +95,6 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default(): global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths - assert global_arguments.excludes_filename is None assert global_arguments.verbosity == 0 assert global_arguments.syslog_verbosity == 0 assert global_arguments.log_file_verbosity == -1 @@ -234,13 +230,6 @@ def test_parse_arguments_disallows_invalid_argument(): module.parse_arguments('--posix-me-harder') -def test_parse_arguments_disallows_deprecated_excludes_option(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes') - - def test_parse_arguments_disallows_encryption_mode_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 87428ddc..1abc3ba5 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -46,7 +46,7 @@ def test_parse_configuration_transforms_file_into_mapping(): - /etc repositories: - - hostname.borg + - path: hostname.borg retention: keep_minutely: 60 @@ -83,7 +83,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): - "/home/{escaped_punctuation}" repositories: - - test.borg + - path: test.borg ''' ) @@ -106,7 +106,7 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): - /home repositories: - - hostname.borg + - path: hostname.borg ''', ''' map: @@ -135,7 +135,7 @@ def test_parse_configuration_inlines_include(): - /home repositories: - - hostname.borg + - path: hostname.borg retention: !include include.yaml @@ -168,7 +168,7 @@ def test_parse_configuration_merges_include(): - /home repositories: - - hostname.borg + - path: hostname.borg retention: keep_daily: 1 @@ -221,7 +221,7 @@ def test_parse_configuration_raises_for_validation_error(): location: source_directories: yes repositories: - - hostname.borg + - path: hostname.borg ''' ) @@ -237,7 +237,7 @@ def test_parse_configuration_applies_overrides(): - /home repositories: - - hostname.borg + - path: hostname.borg local_path: borg1 ''' @@ -265,7 +265,7 @@ def test_parse_configuration_applies_normalization(): - /home repositories: - - hostname.borg + - path: hostname.borg exclude_if_present: .nobackup ''' @@ -280,4 +280,4 @@ def test_parse_configuration_applies_normalization(): 'exclude_if_present': ['.nobackup'], } } - assert logs == [] + assert logs diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 63e3187a..6393d902 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -9,7 +9,7 @@ from borgmatic.config import normalize as module ( {'location': {'exclude_if_present': '.nobackup'}}, {'location': {'exclude_if_present': ['.nobackup']}}, - False, + True, ), ( {'location': {'exclude_if_present': ['.nobackup']}}, @@ -39,22 +39,22 @@ from borgmatic.config import normalize as module ( {'hooks': {'healthchecks': 'https://example.com'}}, {'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}}, - False, + True, ), ( {'hooks': {'cronitor': 'https://example.com'}}, {'hooks': {'cronitor': {'ping_url': 'https://example.com'}}}, - False, + True, ), ( {'hooks': {'pagerduty': 'https://example.com'}}, {'hooks': {'pagerduty': {'integration_key': 'https://example.com'}}}, - False, + True, ), ( {'hooks': {'cronhub': 'https://example.com'}}, {'hooks': {'cronhub': {'ping_url': 'https://example.com'}}}, - False, + True, ), ( {'hooks': None}, @@ -64,12 +64,12 @@ from borgmatic.config import normalize as module ( {'consistency': {'checks': ['archives']}}, {'consistency': {'checks': [{'name': 'archives'}]}}, - False, + True, ), ( {'consistency': {'checks': ['archives']}}, {'consistency': {'checks': [{'name': 'archives'}]}}, - False, + True, ), ( {'consistency': None}, @@ -79,17 +79,17 @@ from borgmatic.config import normalize as module ( {'location': {'numeric_owner': False}}, {'location': {'numeric_ids': False}}, - False, + True, ), ( {'location': {'bsd_flags': False}}, {'location': {'flags': False}}, - False, + True, ), ( {'storage': {'remote_rate_limit': False}}, {'storage': {'upload_rate_limit': False}}, - False, + True, ), ( {'location': {'repositories': ['foo@bar:/repo']}}, @@ -109,12 +109,12 @@ from borgmatic.config import normalize as module ( {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, {'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo'}]}}, - False, + True, ), ( {'location': {'repositories': ['file:///repo']}}, {'location': {'repositories': [{'path': '/repo'}]}}, - False, + True, ), ( {'location': {'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}]}}, @@ -131,6 +131,16 @@ from borgmatic.config import normalize as module {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, False, ), + ( + {'consistency': {'prefix': 'foo'}}, + {'consistency': {'prefix': 'foo'}}, + True, + ), + ( + {'retention': {'prefix': 'foo'}}, + {'retention': {'prefix': 'foo'}}, + True, + ), ), ) def test_normalize_applies_hard_coded_normalization_to_config( From 13a37a1d9bf9cd85a12de2a26c9b61f4441e548d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 30 Jun 2023 22:55:47 -0700 Subject: [PATCH 313/344] Reddit is dead. --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 92b59b2e..f3278d3a 100644 --- a/README.md +++ b/README.md @@ -121,12 +121,7 @@ issues. ### Social -~~Check out the [Borg subreddit](https://www.reddit.com/r/BorgBackup/) for -general Borg and borgmatic discussion and support.~~ borgmatic supports the -ongoing [Reddit user -protests](https://www.theverge.com/2023/6/10/23756476/reddit-protest-api-changes-apollo-third-party-apps). - -Also follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic). +Follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic). ### Chat From fbbfc684ce6f291fa785703e4e1c164dcaf61f9d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 2 Jul 2023 22:14:36 -0700 Subject: [PATCH 314/344] Add referral link for Hetzner. --- README.md | 9 ++++----- docs/how-to/set-up-backups.md | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f3278d3a..78691195 100644 --- a/README.md +++ b/README.md @@ -91,16 +91,15 @@ reference guides. Need somewhere to store your encrypted off-site backups? The following hosting providers include specific support for Borg/borgmatic—and fund borgmatic -development and hosting when you use these links to sign up. (These are -referral links, but without any tracking scripts or cookies.) +development and hosting when you use these referral links to sign up:
  • BorgBase: Borg hosting service with support for monitoring, 2FA, and append-only repos
    • +
  • Hetzner: A "storage box" that includes support for Borg
-Additionally, [rsync.net](https://www.rsync.net/products/borg.html) and -[Hetzner](https://www.hetzner.com/storage/storage-box) have compatible storage -offerings, but do not currently fund borgmatic development or hosting. +Additionally, rsync.net has a compatible storage offering, but does not fund +borgmatic development or hosting. ## Support and contributing diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 7fe69fd9..043817ad 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -104,16 +104,15 @@ installing borgmatic: Need somewhere to store your encrypted off-site backups? The following hosting providers include specific support for Borg/borgmatic—and fund borgmatic -development and hosting when you use these links to sign up. (These are -referral links, but without any tracking scripts or cookies.) +development and hosting when you use these referral links to sign up:
  • BorgBase: Borg hosting service with support for monitoring, 2FA, and append-only repos
    • +
  • Hetzner: A "storage box" that includes support for Borg
-Additionally, [rsync.net](https://www.rsync.net/products/borg.html) and -[Hetzner](https://www.hetzner.com/storage/storage-box) have compatible storage -offerings, but do not currently fund borgmatic development or hosting. +Additionally, rsync.net has a compatible storage offering, but does not fund +borgmatic development or hosting. ## Configuration From 9cafc16052af3002714f97836d90e281e7a7fa70 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 3 Jul 2023 00:08:54 -0700 Subject: [PATCH 315/344] For "borgmatic borg", pass the repository to Borg via a Borg-supported environment variable (#575). --- NEWS | 7 ++- borgmatic/borg/borg.py | 3 +- docs/how-to/run-arbitrary-borg-commands.md | 69 ++++++++++++---------- tests/unit/borg/test_borg.py | 68 ++++++++++----------- 4 files changed, 78 insertions(+), 69 deletions(-) diff --git a/NEWS b/NEWS index 1af50d8f..f1ecea0d 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ 1.8.0.dev0 * #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting - repository/archive into the resulting Borg command-line, make repository and archive environment - variables available for explicit use in your commands. See the documentation for more - information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/ + repository/archive into the resulting Borg command-line, pass repository to Borg via an + environment variable and make archive available for explicit use in your commands. See the + documentation for more information: + https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/ * #719: Fix an error when running "borg key export" through borgmatic. * #720: Fix an error when dumping a MySQL database and the "exclude_nodump" option is set. * When merging two configuration files, error gracefully if the two files do not adhere to the same diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index 107f140a..e0a56923 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -8,7 +8,6 @@ from borgmatic.execute import DO_NOT_CAPTURE, execute_command logger = logging.getLogger(__name__) -REPOSITORYLESS_BORG_COMMANDS = {'serve', None} BORG_SUBCOMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'} @@ -64,7 +63,7 @@ def run_arbitrary_borg( extra_environment=dict( (environment.make_environment(storage_config) or {}), **{ - 'REPOSITORY': repository_path, + 'BORG_REPO': repository_path, 'ARCHIVE': archive if archive else '', }, ), diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index e5720381..8962b908 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -33,57 +33,65 @@ arguments: New in version 1.5.15 The way you run Borg with borgmatic is via the `borg` action. Here's a simple example: -```bash -borgmatic borg break-lock '$REPOSITORY' -``` - -This runs Borg's `break-lock` command once on each configured borgmatic -repository, passing the repository path in as an environment variable named -`REPOSITORY`. The single quotes are necessary in order to pass in a literal -`$REPOSITORY` string instead of trying to resolve it from borgmatic's shell -where it's not yet set. - -Prior to version 1.8.0borgmatic -provided the repository name implicitly, attempting to inject it into your -Borg arguments in the right place (which didn't always work). So your -command-line in these older versions looked more like: - ```bash borgmatic borg break-lock ``` -You can also specify Borg options for relevant commands. In borgmatic 1.8.0+, -that looks like: +This runs Borg's `break-lock` command once with each configured borgmatic +repository, passing the repository path in as a Borg-supported environment +variable named `BORG_REPO`. (The native `borgmatic break-lock` action should +be preferred though for most uses.) + +You can also specify Borg options for relevant commands. For instance: ```bash -borgmatic borg rlist --short '$REPOSITORY' +borgmatic borg rlist --short ``` This runs Borg's `rlist` command once on each configured borgmatic repository. -However, the native `borgmatic rlist` action should be preferred for most uses. What if you only want to run Borg on a single configured borgmatic repository when you've got several configured? Not a problem. The `--repository` argument lets you specify the repository to use, either by its path or its label: ```bash -borgmatic borg --repository repo.borg break-lock '$REPOSITORY' +borgmatic borg --repository repo.borg break-lock ``` +And if you need to specify where the repository goes in the command because +there are positional arguments after it: + +```bash +borgmatic borg debug dump-manifest :: root +``` + +The `::` is a Borg placeholder that means: Substitute the repository passed in +by environment variable here. + +Prior to version 1.8.0borgmatic +attempted to inject the repository name directly into your Borg arguments in +the right place (which didn't always work). So your command-line in these +older versions didn't support the `::` + + ### Specifying an archive For borg commands that expect an archive name, you have a few approaches. Here's one: ```bash -borgmatic borg --archive latest list '$REPOSITORY::$ARCHIVE' +borgmatic borg --archive latest list '::$ARCHIVE' ``` +The single quotes are necessary in order to pass in a literal `$ARCHIVE` +string instead of trying to resolve it from borgmatic's shell where it's not +yet set. + Or if you don't need borgmatic to resolve an archive name like `latest`, you can just do: ```bash -borgmatic borg list '$REPOSITORY::your-actual-archive-name' +borgmatic borg list ::your-actual-archive-name ``` Prior to version 1.8.0borgmatic @@ -100,11 +108,11 @@ borgmatic borg --archive latest list these will list an archive: ```bash -borgmatic borg --archive latest list --repo '$REPOSITORY' '$ARCHIVE' +borgmatic borg --archive latest list '$ARCHIVE' ``` ```bash -borgmatic borg list --repo '$REPOSITORY' your-actual-archive-name +borgmatic borg list your-actual-archive-name ``` ### Limitations @@ -126,12 +134,13 @@ borgmatic's `borg` action is not without limitations: * Unlike normal borgmatic actions that support JSON, the `borg` action will not disable certain borgmatic logs to avoid interfering with JSON output. * Prior to version 1.8.0 - borgmatic implicitly supplied the repository/archive name to Borg for you - (based on your borgmatic configuration or the - `borgmatic borg --repository`/`--archive` arguments)—which meant you couldn't - specify the repository/archive directly in the Borg command. Also, in these - older versions of borgmatic, the `borg` action didn't work for any Borg - commands like `borg serve` that do not accept a repository/archive name. + borgmatic implicitly injected the repository/archive arguments on the Borg + command-line for you (based on your borgmatic configuration or the + `borgmatic borg --repository`/`--archive` arguments)—which meant you + couldn't specify the repository/archive directly in the Borg command. Also, + in these older versions of borgmatic, the `borg` action didn't work for any + Borg commands like `borg serve` that do not accept a repository/archive + name. * Prior to version 1.7.13 Unlike other borgmatic actions, the `borg` action captured (and logged) all output, so interactive prompts and flags like `--progress` dit not work as expected. diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index f03bb89f..2d7e1750 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -13,18 +13,18 @@ def test_run_arbitrary_borg_calls_borg_with_flags(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '$REPOSITORY'), + ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], ) @@ -34,11 +34,11 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '--info', '$REPOSITORY'), + ('borg', 'break-lock', '--info', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.INFO) @@ -46,7 +46,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], ) @@ -56,11 +56,11 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '--debug', '--show-rc', '$REPOSITORY'), + ('borg', 'break-lock', '--debug', '--show-rc', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.DEBUG) @@ -68,7 +68,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], ) @@ -81,18 +81,18 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '--lock-wait', '5', '$REPOSITORY'), + ('borg', 'break-lock', '--lock-wait', '5', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], ) @@ -102,18 +102,18 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '$REPOSITORY::$ARCHIVE'), + ('borg', 'break-lock', '::$ARCHIVE'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': 'archive'}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY::$ARCHIVE'], + options=['break-lock', '::$ARCHIVE'], archive='archive', ) @@ -124,18 +124,18 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg1', 'break-lock', '$REPOSITORY'), + ('borg1', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg1', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], local_path='borg1', ) @@ -148,18 +148,18 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags() ).and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '--remote-path', 'borg1', '$REPOSITORY'), + ('borg', 'break-lock', '--remote-path', 'borg1', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], remote_path='borg1', ) @@ -170,18 +170,18 @@ def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', '--progress', '$REPOSITORY'), + ('borg', 'list', '--progress', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['list', '--progress', '$REPOSITORY'], + options=['list', '--progress', '::'], ) @@ -191,18 +191,18 @@ def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '$REPOSITORY'), + ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['--', 'break-lock', '$REPOSITORY'], + options=['--', 'break-lock', '::'], ) @@ -216,7 +216,7 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( @@ -233,11 +233,11 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'key', 'export', '--info', '$REPOSITORY'), + ('borg', 'key', 'export', '--info', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.INFO) @@ -245,7 +245,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['key', 'export', '$REPOSITORY'], + options=['key', 'export', '::'], ) @@ -255,11 +255,11 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'debug', 'dump-manifest', '--info', '$REPOSITORY', 'path'), + ('borg', 'debug', 'dump-manifest', '--info', '::', 'path'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.INFO) @@ -267,5 +267,5 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['debug', 'dump-manifest', '$REPOSITORY', 'path'], + options=['debug', 'dump-manifest', '::', 'path'], ) From 48b6a1679a20d46d797ffb5f534d33a6bb29931f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 8 Jul 2023 23:14:30 -0700 Subject: [PATCH 316/344] Remove sections (#721). --- NEWS | 7 + borgmatic/actions/borg.py | 6 +- borgmatic/actions/break_lock.py | 4 +- borgmatic/actions/check.py | 17 +- borgmatic/actions/compact.py | 14 +- borgmatic/actions/config/generate.py | 2 + borgmatic/actions/config/validate.py | 3 + borgmatic/actions/create.py | 32 +- borgmatic/actions/export_tar.py | 6 +- borgmatic/actions/extract.py | 17 +- borgmatic/actions/info.py | 6 +- borgmatic/actions/list.py | 6 +- borgmatic/actions/mount.py | 6 +- borgmatic/actions/prune.py | 15 +- borgmatic/actions/rcreate.py | 4 +- borgmatic/actions/restore.py | 74 +- borgmatic/actions/rinfo.py | 4 +- borgmatic/actions/rlist.py | 4 +- borgmatic/actions/transfer.py | 4 +- borgmatic/borg/borg.py | 8 +- borgmatic/borg/break_lock.py | 12 +- borgmatic/borg/check.py | 116 +- borgmatic/borg/compact.py | 14 +- borgmatic/borg/create.py | 111 +- borgmatic/borg/environment.py | 8 +- borgmatic/borg/export_tar.py | 16 +- borgmatic/borg/extract.py | 23 +- borgmatic/borg/info.py | 14 +- borgmatic/borg/list.py | 30 +- borgmatic/borg/mount.py | 8 +- borgmatic/borg/prune.py | 34 +- borgmatic/borg/rcreate.py | 16 +- borgmatic/borg/rinfo.py | 8 +- borgmatic/borg/rlist.py | 26 +- borgmatic/borg/transfer.py | 12 +- borgmatic/borg/version.py | 6 +- borgmatic/commands/arguments.py | 2 +- borgmatic/commands/borgmatic.py | 117 +- borgmatic/config/generate.py | 27 +- borgmatic/config/load.py | 10 +- borgmatic/config/normalize.py | 119 +- borgmatic/config/override.py | 28 +- borgmatic/config/schema.yaml | 2577 ++++++++++----------- borgmatic/config/validate.py | 25 +- borgmatic/hooks/cronhub.py | 2 +- borgmatic/hooks/cronitor.py | 2 +- borgmatic/hooks/dispatch.py | 39 +- borgmatic/hooks/healthchecks.py | 2 +- borgmatic/hooks/mongodb.py | 41 +- borgmatic/hooks/mysql.py | 41 +- borgmatic/hooks/ntfy.py | 2 +- borgmatic/hooks/pagerduty.py | 2 +- borgmatic/hooks/postgresql.py | 48 +- borgmatic/hooks/sqlite.py | 34 +- tests/integration/borg/test_commands.py | 1 - tests/integration/config/test_generate.py | 28 +- tests/integration/config/test_validate.py | 149 +- tests/unit/actions/test_borg.py | 2 +- tests/unit/actions/test_break_lock.py | 2 +- tests/unit/actions/test_check.py | 18 +- tests/unit/actions/test_compact.py | 12 +- tests/unit/actions/test_create.py | 17 +- tests/unit/actions/test_export_tar.py | 2 +- tests/unit/actions/test_extract.py | 4 +- tests/unit/actions/test_info.py | 2 +- tests/unit/actions/test_list.py | 2 +- tests/unit/actions/test_mount.py | 2 +- tests/unit/actions/test_prune.py | 12 +- tests/unit/actions/test_rcreate.py | 4 +- tests/unit/actions/test_restore.py | 87 +- tests/unit/actions/test_rinfo.py | 2 +- tests/unit/actions/test_rlist.py | 2 +- tests/unit/actions/test_transfer.py | 2 +- tests/unit/borg/test_borg.py | 26 +- tests/unit/borg/test_break_lock.py | 14 +- tests/unit/borg/test_check.py | 113 +- tests/unit/borg/test_compact.py | 30 +- tests/unit/borg/test_create.py | 191 +- tests/unit/borg/test_export_tar.py | 28 +- tests/unit/borg/test_extract.py | 67 +- tests/unit/borg/test_info.py | 38 +- tests/unit/borg/test_list.py | 60 +- tests/unit/borg/test_mount.py | 26 +- tests/unit/borg/test_prune.py | 88 +- tests/unit/borg/test_rcreate.py | 34 +- tests/unit/borg/test_rinfo.py | 24 +- tests/unit/borg/test_rlist.py | 58 +- tests/unit/borg/test_transfer.py | 32 +- tests/unit/commands/test_borgmatic.py | 176 +- tests/unit/config/test_generate.py | 35 +- tests/unit/config/test_normalize.py | 313 ++- tests/unit/config/test_override.py | 46 +- tests/unit/config/test_validate.py | 56 +- tests/unit/hooks/test_cronhub.py | 23 +- tests/unit/hooks/test_cronitor.py | 22 +- tests/unit/hooks/test_dispatch.py | 44 +- tests/unit/hooks/test_healthchecks.py | 12 + tests/unit/hooks/test_mongodb.py | 36 +- tests/unit/hooks/test_mysql.py | 28 +- tests/unit/hooks/test_ntfy.py | 12 + tests/unit/hooks/test_pagerduty.py | 6 + tests/unit/hooks/test_postgresql.py | 48 +- tests/unit/hooks/test_sqlite.py | 20 +- 103 files changed, 2817 insertions(+), 3050 deletions(-) diff --git a/NEWS b/NEWS index f1ecea0d..2831f29a 100644 --- a/NEWS +++ b/NEWS @@ -8,6 +8,13 @@ * #720: Fix an error when dumping a MySQL database and the "exclude_nodump" option is set. * When merging two configuration files, error gracefully if the two files do not adhere to the same format. + * #721: Remove configuration sections ("location:", "storage:", "hooks:" etc.), while still keeping + deprecated support for them. Now, all options are at the same level, and you don't need to worry + about commenting/uncommenting section headers when you change an option. + * #721: BREAKING: The retention prefix and the consistency prefix can no longer have different + values (unless one is not set). + * #721: BREAKING: The storage umask and the hooks umask can no longer have different values (unless + one is not set). * BREAKING: Remove the deprecated (and silently ignored) "--successful" flag on the "list" action, as newer versions of Borg list successful (non-checkpoint) archives by default. * All deprecated configuration option values now generate warning logs. diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index 44ffc951..6e46596f 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def run_borg( repository, - storage, + config, local_borg_version, borg_arguments, global_arguments, @@ -28,7 +28,7 @@ def run_borg( archive_name = borgmatic.borg.rlist.resolve_archive_name( repository['path'], borg_arguments.archive, - storage, + config, local_borg_version, global_arguments, local_path, @@ -36,7 +36,7 @@ def run_borg( ) borgmatic.borg.borg.run_arbitrary_borg( repository['path'], - storage, + config, local_borg_version, options=borg_arguments.options, archive=archive_name, diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index a00d5785..e94ab466 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def run_break_lock( repository, - storage, + config, local_borg_version, break_lock_arguments, global_arguments, @@ -26,7 +26,7 @@ def run_break_lock( ) borgmatic.borg.break_lock.break_lock( repository['path'], - storage, + config, local_borg_version, global_arguments, local_path=local_path, diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 610d41ee..fe800156 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -10,10 +10,7 @@ logger = logging.getLogger(__name__) def run_check( config_filename, repository, - location, - storage, - consistency, - hooks, + config, hook_context, local_borg_version, check_arguments, @@ -30,8 +27,8 @@ def run_check( return borgmatic.hooks.command.execute_hook( - hooks.get('before_check'), - hooks.get('umask'), + config.get('before_check'), + config.get('umask'), config_filename, 'pre-check', global_arguments.dry_run, @@ -40,9 +37,7 @@ def run_check( logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks') borgmatic.borg.check.check_archives( repository['path'], - location, - storage, - consistency, + config, local_borg_version, global_arguments, local_path=local_path, @@ -53,8 +48,8 @@ def run_check( force=check_arguments.force, ) borgmatic.hooks.command.execute_hook( - hooks.get('after_check'), - hooks.get('umask'), + config.get('after_check'), + config.get('umask'), config_filename, 'post-check', global_arguments.dry_run, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index ad680d21..29cf8943 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -11,9 +11,7 @@ logger = logging.getLogger(__name__) def run_compact( config_filename, repository, - storage, - retention, - hooks, + config, hook_context, local_borg_version, compact_arguments, @@ -31,8 +29,8 @@ def run_compact( return borgmatic.hooks.command.execute_hook( - hooks.get('before_compact'), - hooks.get('umask'), + config.get('before_compact'), + config.get('umask'), config_filename, 'pre-compact', global_arguments.dry_run, @@ -45,7 +43,7 @@ def run_compact( borgmatic.borg.compact.compact_segments( global_arguments.dry_run, repository['path'], - storage, + config, local_borg_version, global_arguments, local_path=local_path, @@ -59,8 +57,8 @@ def run_compact( f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)' ) borgmatic.hooks.command.execute_hook( - hooks.get('after_compact'), - hooks.get('umask'), + config.get('after_compact'), + config.get('umask'), config_filename, 'post-compact', global_arguments.dry_run, diff --git a/borgmatic/actions/config/generate.py b/borgmatic/actions/config/generate.py index 1943ea74..48b9a7dd 100644 --- a/borgmatic/actions/config/generate.py +++ b/borgmatic/actions/config/generate.py @@ -2,6 +2,7 @@ import logging import borgmatic.config.generate import borgmatic.config.validate +import borgmatic.logger logger = logging.getLogger(__name__) @@ -14,6 +15,7 @@ def run_generate(generate_arguments, global_arguments): Raise FileExistsError if a file already exists at the destination path and the generate arguments do not have overwrite set. ''' + borgmatic.logger.add_custom_log_levels() dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else '' logger.answer( diff --git a/borgmatic/actions/config/validate.py b/borgmatic/actions/config/validate.py index 2cec6135..2929ccaa 100644 --- a/borgmatic/actions/config/validate.py +++ b/borgmatic/actions/config/validate.py @@ -1,6 +1,7 @@ import logging import borgmatic.config.generate +import borgmatic.logger logger = logging.getLogger(__name__) @@ -14,6 +15,8 @@ def run_validate(validate_arguments, configs): loading machinery prior to here, so this function mainly exists to support additional validate flags like "--show". ''' + borgmatic.logger.add_custom_log_levels() + if validate_arguments.show: for config_path, config in configs.items(): if len(configs) > 1: diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 1bacf73b..1d750f62 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -17,7 +17,7 @@ import borgmatic.hooks.dump logger = logging.getLogger(__name__) -def create_borgmatic_manifest(location, config_paths, dry_run): +def create_borgmatic_manifest(config, config_paths, dry_run): ''' Create a borgmatic manifest file to store the paths to the configuration files used to create the archive. @@ -25,7 +25,7 @@ def create_borgmatic_manifest(location, config_paths, dry_run): if dry_run: return - borgmatic_source_directory = location.get( + borgmatic_source_directory = config.get( 'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY ) @@ -49,9 +49,7 @@ def create_borgmatic_manifest(location, config_paths, dry_run): def run_create( config_filename, repository, - location, - storage, - hooks, + config, hook_context, local_borg_version, create_arguments, @@ -71,8 +69,8 @@ def run_create( return borgmatic.hooks.command.execute_hook( - hooks.get('before_backup'), - hooks.get('umask'), + config.get('before_backup'), + config.get('umask'), config_filename, 'pre-backup', global_arguments.dry_run, @@ -81,30 +79,25 @@ def run_create( logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}') borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', - hooks, + config, repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, - location, global_arguments.dry_run, ) active_dumps = borgmatic.hooks.dispatch.call_hooks( 'dump_databases', - hooks, + config, repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, - location, global_arguments.dry_run, ) - create_borgmatic_manifest( - location, global_arguments.used_config_paths, global_arguments.dry_run - ) + create_borgmatic_manifest(config, global_arguments.used_config_paths, global_arguments.dry_run) stream_processes = [process for processes in active_dumps.values() for process in processes] json_output = borgmatic.borg.create.create_archive( global_arguments.dry_run, repository['path'], - location, - storage, + config, local_borg_version, global_arguments, local_path=local_path, @@ -120,15 +113,14 @@ def run_create( borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', - hooks, + config, config_filename, borgmatic.hooks.dump.DATABASE_HOOK_NAMES, - location, global_arguments.dry_run, ) borgmatic.hooks.command.execute_hook( - hooks.get('after_backup'), - hooks.get('umask'), + config.get('after_backup'), + config.get('umask'), config_filename, 'post-backup', global_arguments.dry_run, diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index 798bd418..f1937b54 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def run_export_tar( repository, - storage, + config, local_borg_version, export_tar_arguments, global_arguments, @@ -31,7 +31,7 @@ def run_export_tar( borgmatic.borg.rlist.resolve_archive_name( repository['path'], export_tar_arguments.archive, - storage, + config, local_borg_version, global_arguments, local_path, @@ -39,7 +39,7 @@ def run_export_tar( ), export_tar_arguments.paths, export_tar_arguments.destination, - storage, + config, local_borg_version, global_arguments, local_path=local_path, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 0bb1efb7..5d02d4c7 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -11,9 +11,7 @@ logger = logging.getLogger(__name__) def run_extract( config_filename, repository, - location, - storage, - hooks, + config, hook_context, local_borg_version, extract_arguments, @@ -25,8 +23,8 @@ def run_extract( Run the "extract" action for the given repository. ''' borgmatic.hooks.command.execute_hook( - hooks.get('before_extract'), - hooks.get('umask'), + config.get('before_extract'), + config.get('umask'), config_filename, 'pre-extract', global_arguments.dry_run, @@ -44,15 +42,14 @@ def run_extract( borgmatic.borg.rlist.resolve_archive_name( repository['path'], extract_arguments.archive, - storage, + config, local_borg_version, global_arguments, local_path, remote_path, ), extract_arguments.paths, - location, - storage, + config, local_borg_version, global_arguments, local_path=local_path, @@ -62,8 +59,8 @@ def run_extract( progress=extract_arguments.progress, ) borgmatic.hooks.command.execute_hook( - hooks.get('after_extract'), - hooks.get('umask'), + config.get('after_extract'), + config.get('umask'), config_filename, 'post-extract', global_arguments.dry_run, diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index 0116fd70..b09f3ece 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) def run_info( repository, - storage, + config, local_borg_version, info_arguments, global_arguments, @@ -33,7 +33,7 @@ def run_info( archive_name = borgmatic.borg.rlist.resolve_archive_name( repository['path'], info_arguments.archive, - storage, + config, local_borg_version, global_arguments, local_path, @@ -41,7 +41,7 @@ def run_info( ) json_output = borgmatic.borg.info.display_archives_info( repository['path'], - storage, + config, local_borg_version, borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name), global_arguments, diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 667062d8..ae9da63c 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) def run_list( repository, - storage, + config, local_borg_version, list_arguments, global_arguments, @@ -34,7 +34,7 @@ def run_list( archive_name = borgmatic.borg.rlist.resolve_archive_name( repository['path'], list_arguments.archive, - storage, + config, local_borg_version, global_arguments, local_path, @@ -42,7 +42,7 @@ def run_list( ) json_output = borgmatic.borg.list.list_archive( repository['path'], - storage, + config, local_borg_version, borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name), global_arguments, diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index cc8a2cbd..86b05859 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def run_mount( repository, - storage, + config, local_borg_version, mount_arguments, global_arguments, @@ -34,14 +34,14 @@ def run_mount( borgmatic.borg.rlist.resolve_archive_name( repository['path'], mount_arguments.archive, - storage, + config, local_borg_version, global_arguments, local_path, remote_path, ), mount_arguments, - storage, + config, local_borg_version, global_arguments, local_path=local_path, diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index deaea384..0cb074b5 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -10,9 +10,7 @@ logger = logging.getLogger(__name__) def run_prune( config_filename, repository, - storage, - retention, - hooks, + config, hook_context, local_borg_version, prune_arguments, @@ -30,8 +28,8 @@ def run_prune( return borgmatic.hooks.command.execute_hook( - hooks.get('before_prune'), - hooks.get('umask'), + config.get('before_prune'), + config.get('umask'), config_filename, 'pre-prune', global_arguments.dry_run, @@ -41,8 +39,7 @@ def run_prune( borgmatic.borg.prune.prune_archives( global_arguments.dry_run, repository['path'], - storage, - retention, + config, local_borg_version, prune_arguments, global_arguments, @@ -50,8 +47,8 @@ def run_prune( remote_path=remote_path, ) borgmatic.hooks.command.execute_hook( - hooks.get('after_prune'), - hooks.get('umask'), + config.get('after_prune'), + config.get('umask'), config_filename, 'post-prune', global_arguments.dry_run, diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index 1bfc489b..32cdef40 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def run_rcreate( repository, - storage, + config, local_borg_version, rcreate_arguments, global_arguments, @@ -27,7 +27,7 @@ def run_rcreate( borgmatic.borg.rcreate.create_repository( global_arguments.dry_run, repository['path'], - storage, + config, local_borg_version, global_arguments, rcreate_arguments.encryption_mode, diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index d44a2cac..8112207a 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -18,12 +18,12 @@ UNSPECIFIED_HOOK = object() def get_configured_database( - hooks, archive_database_names, hook_name, database_name, configuration_database_name=None + config, archive_database_names, hook_name, database_name, configuration_database_name=None ): ''' - Find the first database with the given hook name and database name in the configured hooks - dict and the given archive database names dict (from hook name to database names contained in - a particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all database + Find the first database with the given hook name and database name in the configuration dict and + the given archive database names dict (from hook name to database names contained in a + particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all database hooks for the named database. If a configuration database name is given, use that instead of the database name to lookup the database in the given hooks configuration. @@ -33,9 +33,13 @@ def get_configured_database( configuration_database_name = database_name if hook_name == UNSPECIFIED_HOOK: - hooks_to_search = hooks + hooks_to_search = { + hook_name: value + for (hook_name, value) in config.items() + if hook_name in borgmatic.hooks.dump.DATABASE_HOOK_NAMES + } else: - hooks_to_search = {hook_name: hooks[hook_name]} + hooks_to_search = {hook_name: config[hook_name]} return next( ( @@ -58,9 +62,7 @@ def get_configured_hook_name_and_database(hooks, database_name): def restore_single_database( repository, - location, - storage, - hooks, + config, local_borg_version, global_arguments, local_path, @@ -81,10 +83,9 @@ def restore_single_database( dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', - hooks, + config, repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, - location, database['name'], )[hook_name] @@ -94,8 +95,7 @@ def restore_single_database( repository=repository['path'], archive=archive_name, paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]), - location_config=location, - storage_config=storage, + config=config, local_borg_version=local_borg_version, global_arguments=global_arguments, local_path=local_path, @@ -112,7 +112,7 @@ def restore_single_database( {hook_name: [database]}, repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, - location, + config, global_arguments.dry_run, extract_process, connection_params, @@ -122,21 +122,20 @@ def restore_single_database( def collect_archive_database_names( repository, archive, - location, - storage, + config, local_borg_version, global_arguments, local_path, remote_path, ): ''' - Given a local or remote repository path, a resolved archive name, a location configuration dict, - a storage configuration dict, the local Borg version, global_arguments an argparse.Namespace, - and local and remote Borg paths, query the archive for the names of databases it contains and - return them as a dict from hook name to a sequence of database names. + Given a local or remote repository path, a resolved archive name, a configuration dict, the + local Borg version, global_arguments an argparse.Namespace, and local and remote Borg paths, + query the archive for the names of databases it contains and return them as a dict from hook + name to a sequence of database names. ''' borgmatic_source_directory = os.path.expanduser( - location.get( + config.get( 'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY ) ).lstrip('/') @@ -146,7 +145,7 @@ def collect_archive_database_names( dump_paths = borgmatic.borg.list.capture_archive_listing( repository, archive, - storage, + config, local_borg_version, global_arguments, list_path=parent_dump_path, @@ -249,9 +248,7 @@ def ensure_databases_found(restore_names, remaining_restore_names, found_names): def run_restore( repository, - location, - storage, - hooks, + config, local_borg_version, restore_arguments, global_arguments, @@ -275,17 +272,16 @@ def run_restore( borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', - hooks, + config, repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, - location, global_arguments.dry_run, ) archive_name = borgmatic.borg.rlist.resolve_archive_name( repository['path'], restore_arguments.archive, - storage, + config, local_borg_version, global_arguments, local_path, @@ -294,8 +290,7 @@ def run_restore( archive_database_names = collect_archive_database_names( repository['path'], archive_name, - location, - storage, + config, local_borg_version, global_arguments, local_path, @@ -315,7 +310,7 @@ def run_restore( for hook_name, database_names in restore_names.items(): for database_name in database_names: found_hook_name, found_database = get_configured_database( - hooks, archive_database_names, hook_name, database_name + config, archive_database_names, hook_name, database_name ) if not found_database: @@ -327,9 +322,7 @@ def run_restore( found_names.add(database_name) restore_single_database( repository, - location, - storage, - hooks, + config, local_borg_version, global_arguments, local_path, @@ -340,12 +333,12 @@ def run_restore( connection_params, ) - # For any database that weren't found via exact matches in the hooks configuration, try to - # fallback to "all" entries. + # For any database that weren't found via exact matches in the configuration, try to fallback + # to "all" entries. for hook_name, database_names in remaining_restore_names.items(): for database_name in database_names: found_hook_name, found_database = get_configured_database( - hooks, archive_database_names, hook_name, database_name, 'all' + config, archive_database_names, hook_name, database_name, 'all' ) if not found_database: @@ -357,9 +350,7 @@ def run_restore( restore_single_database( repository, - location, - storage, - hooks, + config, local_borg_version, global_arguments, local_path, @@ -372,10 +363,9 @@ def run_restore( borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', - hooks, + config, repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, - location, global_arguments.dry_run, ) diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 7756efd0..00de8922 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def run_rinfo( repository, - storage, + config, local_borg_version, rinfo_arguments, global_arguments, @@ -31,7 +31,7 @@ def run_rinfo( json_output = borgmatic.borg.rinfo.display_repository_info( repository['path'], - storage, + config, local_borg_version, rinfo_arguments=rinfo_arguments, global_arguments=global_arguments, diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index a9dee21d..a79920b6 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def run_rlist( repository, - storage, + config, local_borg_version, rlist_arguments, global_arguments, @@ -29,7 +29,7 @@ def run_rlist( json_output = borgmatic.borg.rlist.list_repository( repository['path'], - storage, + config, local_borg_version, rlist_arguments=rlist_arguments, global_arguments=global_arguments, diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index df481e4d..4051b14e 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) def run_transfer( repository, - storage, + config, local_borg_version, transfer_arguments, global_arguments, @@ -23,7 +23,7 @@ def run_transfer( borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, repository['path'], - storage, + config, local_borg_version, transfer_arguments, global_arguments, diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index e0a56923..1c0d6d1c 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -13,7 +13,7 @@ BORG_SUBCOMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'} def run_arbitrary_borg( repository_path, - storage_config, + config, local_borg_version, options, archive=None, @@ -21,13 +21,13 @@ def run_arbitrary_borg( remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, a + Given a local or remote repository path, a configuration dict, the local Borg version, a sequence of arbitrary command-line Borg options, and an optional archive name, run an arbitrary Borg command, passing in REPOSITORY and ARCHIVE environment variables for optional use in the command. ''' borgmatic.logger.add_custom_log_levels() - lock_wait = storage_config.get('lock_wait', None) + lock_wait = config.get('lock_wait', None) try: options = options[1:] if options[0] == '--' else options @@ -61,7 +61,7 @@ def run_arbitrary_borg( borg_local_path=local_path, shell=True, extra_environment=dict( - (environment.make_environment(storage_config) or {}), + (environment.make_environment(config) or {}), **{ 'BORG_REPO': repository_path, 'ARCHIVE': archive if archive else '', diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 3c361956..c0ee5dbc 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -8,19 +8,19 @@ logger = logging.getLogger(__name__) def break_lock( repository_path, - storage_config, + config, local_borg_version, global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage configuration dict, the local Borg version, - an argparse.Namespace of global arguments, and optional local and remote Borg paths, break any + Given a local or remote repository path, a configuration dict, the local Borg version, an + argparse.Namespace of global arguments, and optional local and remote Borg paths, break any repository and cache locks leftover from Borg aborting. ''' - umask = storage_config.get('umask', None) - lock_wait = storage_config.get('lock_wait', None) + umask = config.get('umask', None) + lock_wait = config.get('lock_wait', None) full_command = ( (local_path, 'break-lock') @@ -33,5 +33,5 @@ def break_lock( + flags.make_repository_flags(repository_path, local_borg_version) ) - borg_environment = environment.make_environment(storage_config) + borg_environment = environment.make_environment(config) execute_command(full_command, borg_local_path=local_path, extra_environment=borg_environment) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 0e54a2cd..859052b0 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -19,12 +19,12 @@ DEFAULT_CHECKS = ( logger = logging.getLogger(__name__) -def parse_checks(consistency_config, only_checks=None): +def parse_checks(config, only_checks=None): ''' - Given a consistency config with a "checks" sequence of dicts and an optional list of override + Given a configuration dict with a "checks" sequence of dicts and an optional list of override checks, return a tuple of named checks to run. - For example, given a retention config of: + For example, given a config of: {'checks': ({'name': 'repository'}, {'name': 'archives'})} @@ -36,8 +36,7 @@ def parse_checks(consistency_config, only_checks=None): has a name of "disabled", return an empty tuple, meaning that no checks should be run. ''' checks = only_checks or tuple( - check_config['name'] - for check_config in (consistency_config.get('checks', None) or DEFAULT_CHECKS) + check_config['name'] for check_config in (config.get('checks', None) or DEFAULT_CHECKS) ) checks = tuple(check.lower() for check in checks) if 'disabled' in checks: @@ -90,23 +89,22 @@ def parse_frequency(frequency): def filter_checks_on_frequency( - location_config, - consistency_config, + config, borg_repository_id, checks, force, archives_check_id=None, ): ''' - Given a location config, a consistency config with a "checks" sequence of dicts, a Borg - repository ID, a sequence of checks, whether to force checks to run, and an ID for the archives - check potentially being run (if any), filter down those checks based on the configured - "frequency" for each check as compared to its check time file. + Given a configuration dict with a "checks" sequence of dicts, a Borg repository ID, a sequence + of checks, whether to force checks to run, and an ID for the archives check potentially being + run (if any), filter down those checks based on the configured "frequency" for each check as + compared to its check time file. In other words, a check whose check time file's timestamp is too new (based on the configured frequency) will get cut from the returned sequence of checks. Example: - consistency_config = { + config = { 'checks': [ { 'name': 'archives', @@ -115,9 +113,9 @@ def filter_checks_on_frequency( ] } - When this function is called with that consistency_config and "archives" in checks, "archives" - will get filtered out of the returned result if its check time file is newer than 2 weeks old, - indicating that it's not yet time to run that check again. + When this function is called with that config and "archives" in checks, "archives" will get + filtered out of the returned result if its check time file is newer than 2 weeks old, indicating + that it's not yet time to run that check again. Raise ValueError if a frequency cannot be parsed. ''' @@ -126,7 +124,7 @@ def filter_checks_on_frequency( if force: return tuple(filtered_checks) - for check_config in consistency_config.get('checks', DEFAULT_CHECKS): + for check_config in config.get('checks', DEFAULT_CHECKS): check = check_config['name'] if checks and check not in checks: continue @@ -135,9 +133,7 @@ def filter_checks_on_frequency( if not frequency_delta: continue - check_time = probe_for_check_time( - location_config, borg_repository_id, check, archives_check_id - ) + check_time = probe_for_check_time(config, borg_repository_id, check, archives_check_id) if not check_time: continue @@ -153,13 +149,11 @@ def filter_checks_on_frequency( return tuple(filtered_checks) -def make_archive_filter_flags( - local_borg_version, storage_config, checks, check_last=None, prefix=None -): +def make_archive_filter_flags(local_borg_version, config, checks, check_last=None, prefix=None): ''' - Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the - check last value, and a consistency check prefix, transform the checks into tuple of - command-line flags for filtering archives in a check command. + Given the local Borg version, a configuration dict, a parsed sequence of checks, the check last + value, and a consistency check prefix, transform the checks into tuple of command-line flags for + filtering archives in a check command. If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if a prefix value is given and "archives" is in checks, then include a "--match-archives" flag. @@ -174,8 +168,8 @@ def make_archive_filter_flags( if prefix else ( flags.make_match_archives_flags( - storage_config.get('match_archives'), - storage_config.get('archive_name_format'), + config.get('match_archives'), + config.get('archive_name_format'), local_borg_version, ) ) @@ -237,14 +231,14 @@ def make_check_flags(checks, archive_filter_flags): ) -def make_check_time_path(location_config, borg_repository_id, check_type, archives_check_id=None): +def make_check_time_path(config, borg_repository_id, check_type, archives_check_id=None): ''' - Given a location configuration dict, a Borg repository ID, the name of a check type - ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a - path for recording that check's time (the time of that check last occurring). + Given a configuration dict, a Borg repository ID, the name of a check type ("repository", + "archives", etc.), and a unique hash of the archives filter flags, return a path for recording + that check's time (the time of that check last occurring). ''' borgmatic_source_directory = os.path.expanduser( - location_config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY) + config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY) ) if check_type in ('archives', 'data'): @@ -287,11 +281,11 @@ def read_check_time(path): return None -def probe_for_check_time(location_config, borg_repository_id, check, archives_check_id): +def probe_for_check_time(config, borg_repository_id, check, archives_check_id): ''' - Given a location configuration dict, a Borg repository ID, the name of a check type - ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a - the corresponding check time or None if such a check time does not exist. + Given a configuration dict, a Borg repository ID, the name of a check type ("repository", + "archives", etc.), and a unique hash of the archives filter flags, return a the corresponding + check time or None if such a check time does not exist. When the check type is "archives" or "data", this function probes two different paths to find the check time, e.g.: @@ -311,8 +305,8 @@ def probe_for_check_time(location_config, borg_repository_id, check, archives_ch read_check_time(group[0]) for group in itertools.groupby( ( - make_check_time_path(location_config, borg_repository_id, check, archives_check_id), - make_check_time_path(location_config, borg_repository_id, check), + make_check_time_path(config, borg_repository_id, check, archives_check_id), + make_check_time_path(config, borg_repository_id, check), ) ) ) @@ -323,10 +317,10 @@ def probe_for_check_time(location_config, borg_repository_id, check, archives_ch return None -def upgrade_check_times(location_config, borg_repository_id): +def upgrade_check_times(config, borg_repository_id): ''' - Given a location configuration dict and a Borg repository ID, upgrade any corresponding check - times on disk from old-style paths to new-style paths. + Given a configuration dict and a Borg repository ID, upgrade any corresponding check times on + disk from old-style paths to new-style paths. Currently, the only upgrade performed is renaming an archive or data check path that looks like: @@ -337,7 +331,7 @@ def upgrade_check_times(location_config, borg_repository_id): ~/.borgmatic/checks/1234567890/archives/all ''' for check_type in ('archives', 'data'): - new_path = make_check_time_path(location_config, borg_repository_id, check_type, 'all') + new_path = make_check_time_path(config, borg_repository_id, check_type, 'all') old_path = os.path.dirname(new_path) temporary_path = f'{old_path}.temp' @@ -357,9 +351,7 @@ def upgrade_check_times(location_config, borg_repository_id): def check_archives( repository_path, - location_config, - storage_config, - consistency_config, + config, local_borg_version, global_arguments, local_path='borg', @@ -370,10 +362,9 @@ def check_archives( force=None, ): ''' - Given a local or remote repository path, a storage config dict, a consistency config dict, - local/remote commands to run, whether to include progress information, whether to attempt a - repair, and an optional list of checks to use instead of configured checks, check the contained - Borg archives for consistency. + Given a local or remote repository path, a configuration dict, local/remote commands to run, + whether to include progress information, whether to attempt a repair, and an optional list of + checks to use instead of configured checks, check the contained Borg archives for consistency. If there are no consistency checks to run, skip running them. @@ -383,7 +374,7 @@ def check_archives( borg_repository_id = json.loads( rinfo.display_repository_info( repository_path, - storage_config, + config, local_borg_version, argparse.Namespace(json=True), global_arguments, @@ -394,21 +385,20 @@ def check_archives( except (json.JSONDecodeError, KeyError): raise ValueError(f'Cannot determine Borg repository ID for {repository_path}') - upgrade_check_times(location_config, borg_repository_id) + upgrade_check_times(config, borg_repository_id) - check_last = consistency_config.get('check_last', None) - prefix = consistency_config.get('prefix') - configured_checks = parse_checks(consistency_config, only_checks) + check_last = config.get('check_last', None) + prefix = config.get('prefix') + configured_checks = parse_checks(config, only_checks) lock_wait = None - extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') + extra_borg_options = config.get('extra_borg_options', {}).get('check', '') archive_filter_flags = make_archive_filter_flags( - local_borg_version, storage_config, configured_checks, check_last, prefix + local_borg_version, config, configured_checks, check_last, prefix ) archives_check_id = make_archives_check_id(archive_filter_flags) checks = filter_checks_on_frequency( - location_config, - consistency_config, + config, borg_repository_id, configured_checks, force, @@ -416,7 +406,7 @@ def check_archives( ) if set(checks).intersection({'repository', 'archives', 'data'}): - lock_wait = storage_config.get('lock_wait') + lock_wait = config.get('lock_wait') verbosity_flags = () if logger.isEnabledFor(logging.INFO): @@ -437,7 +427,7 @@ def check_archives( + flags.make_repository_flags(repository_path, local_borg_version) ) - borg_environment = environment.make_environment(storage_config) + borg_environment = environment.make_environment(config) # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. @@ -450,12 +440,12 @@ def check_archives( for check in checks: write_check_time( - make_check_time_path(location_config, borg_repository_id, check, archives_check_id) + make_check_time_path(config, borg_repository_id, check, archives_check_id) ) if 'extract' in checks: extract.extract_last_archive_dry_run( - storage_config, + config, local_borg_version, global_arguments, repository_path, @@ -463,4 +453,4 @@ def check_archives( local_path, remote_path, ) - write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract')) + write_check_time(make_check_time_path(config, borg_repository_id, 'extract')) diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index 24f37ee3..20bbe129 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def compact_segments( dry_run, repository_path, - storage_config, + config, local_borg_version, global_arguments, local_path='borg', @@ -19,12 +19,12 @@ def compact_segments( threshold=None, ): ''' - Given dry-run flag, a local or remote repository path, a storage config dict, and the local - Borg version, compact the segments in a repository. + Given dry-run flag, a local or remote repository path, a configuration dict, and the local Borg + version, compact the segments in a repository. ''' - umask = storage_config.get('umask', None) - lock_wait = storage_config.get('lock_wait', None) - extra_borg_options = storage_config.get('extra_borg_options', {}).get('compact', '') + umask = config.get('umask', None) + lock_wait = config.get('lock_wait', None) + extra_borg_options = config.get('extra_borg_options', {}).get('compact', '') full_command = ( (local_path, 'compact') @@ -49,5 +49,5 @@ def compact_segments( full_command, output_log_level=logging.INFO, borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), + extra_environment=environment.make_environment(config), ) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 1ec54fdd..018f447d 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -146,12 +146,12 @@ def ensure_files_readable(*filename_lists): open(file_object).close() -def make_pattern_flags(location_config, pattern_filename=None): +def make_pattern_flags(config, pattern_filename=None): ''' - Given a location config dict with a potential patterns_from option, and a filename containing - any additional patterns, return the corresponding Borg flags for those files as a tuple. + Given a configuration dict with a potential patterns_from option, and a filename containing any + additional patterns, return the corresponding Borg flags for those files as a tuple. ''' - pattern_filenames = tuple(location_config.get('patterns_from') or ()) + ( + pattern_filenames = tuple(config.get('patterns_from') or ()) + ( (pattern_filename,) if pattern_filename else () ) @@ -162,12 +162,12 @@ def make_pattern_flags(location_config, pattern_filename=None): ) -def make_exclude_flags(location_config, exclude_filename=None): +def make_exclude_flags(config, exclude_filename=None): ''' - Given a location config dict with various exclude options, and a filename containing any exclude + Given a configuration dict with various exclude options, and a filename containing any exclude patterns, return the corresponding Borg flags as a tuple. ''' - exclude_filenames = tuple(location_config.get('exclude_from') or ()) + ( + exclude_filenames = tuple(config.get('exclude_from') or ()) + ( (exclude_filename,) if exclude_filename else () ) exclude_from_flags = tuple( @@ -175,17 +175,15 @@ def make_exclude_flags(location_config, exclude_filename=None): ('--exclude-from', exclude_filename) for exclude_filename in exclude_filenames ) ) - caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else () + caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else () if_present_flags = tuple( itertools.chain.from_iterable( ('--exclude-if-present', if_present) - for if_present in location_config.get('exclude_if_present', ()) + for if_present in config.get('exclude_if_present', ()) ) ) - keep_exclude_tags_flags = ( - ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else () - ) - exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else () + keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else () + exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else () return ( exclude_from_flags @@ -326,8 +324,7 @@ def check_all_source_directories_exist(source_directories): def create_archive( dry_run, repository_path, - location_config, - storage_config, + config, local_borg_version, global_arguments, local_path='borg', @@ -339,72 +336,70 @@ def create_archive( stream_processes=None, ): ''' - Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a - storage config dict, create a Borg archive and return Borg's JSON output (if any). + Given vebosity/dry-run flags, a local or remote repository path, and a configuration dict, + create a Borg archive and return Borg's JSON output (if any). If a sequence of stream processes is given (instances of subprocess.Popen), then execute the create command while also triggering the given processes to produce output. ''' borgmatic.logger.add_custom_log_levels() borgmatic_source_directories = expand_directories( - collect_borgmatic_source_directories(location_config.get('borgmatic_source_directory')) + collect_borgmatic_source_directories(config.get('borgmatic_source_directory')) ) - if location_config.get('source_directories_must_exist', False): - check_all_source_directories_exist(location_config.get('source_directories')) + if config.get('source_directories_must_exist', False): + check_all_source_directories_exist(config.get('source_directories')) sources = deduplicate_directories( map_directories_to_devices( expand_directories( - tuple(location_config.get('source_directories', ())) + tuple(config.get('source_directories', ())) + borgmatic_source_directories + tuple(global_arguments.used_config_paths) ) ), additional_directory_devices=map_directories_to_devices( - expand_directories(pattern_root_directories(location_config.get('patterns'))) + expand_directories(pattern_root_directories(config.get('patterns'))) ), ) - ensure_files_readable(location_config.get('patterns_from'), location_config.get('exclude_from')) + ensure_files_readable(config.get('patterns_from'), config.get('exclude_from')) try: - working_directory = os.path.expanduser(location_config.get('working_directory')) + working_directory = os.path.expanduser(config.get('working_directory')) except TypeError: working_directory = None pattern_file = ( - write_pattern_file(location_config.get('patterns'), sources) - if location_config.get('patterns') or location_config.get('patterns_from') + write_pattern_file(config.get('patterns'), sources) + if config.get('patterns') or config.get('patterns_from') else None ) - exclude_file = write_pattern_file( - expand_home_directories(location_config.get('exclude_patterns')) - ) - checkpoint_interval = storage_config.get('checkpoint_interval', None) - checkpoint_volume = storage_config.get('checkpoint_volume', None) - chunker_params = storage_config.get('chunker_params', None) - compression = storage_config.get('compression', None) - upload_rate_limit = storage_config.get('upload_rate_limit', None) - umask = storage_config.get('umask', None) - lock_wait = storage_config.get('lock_wait', None) + exclude_file = write_pattern_file(expand_home_directories(config.get('exclude_patterns'))) + checkpoint_interval = config.get('checkpoint_interval', None) + checkpoint_volume = config.get('checkpoint_volume', None) + chunker_params = config.get('chunker_params', None) + compression = config.get('compression', None) + upload_rate_limit = config.get('upload_rate_limit', None) + umask = config.get('umask', None) + lock_wait = config.get('lock_wait', None) list_filter_flags = make_list_filter_flags(local_borg_version, dry_run) - files_cache = location_config.get('files_cache') - archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT) - extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '') + files_cache = config.get('files_cache') + archive_name_format = config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT) + extra_borg_options = config.get('extra_borg_options', {}).get('create', '') if feature.available(feature.Feature.ATIME, local_borg_version): - atime_flags = ('--atime',) if location_config.get('atime') is True else () + atime_flags = ('--atime',) if config.get('atime') is True else () else: - atime_flags = ('--noatime',) if location_config.get('atime') is False else () + atime_flags = ('--noatime',) if config.get('atime') is False else () if feature.available(feature.Feature.NOFLAGS, local_borg_version): - noflags_flags = ('--noflags',) if location_config.get('flags') is False else () + noflags_flags = ('--noflags',) if config.get('flags') is False else () else: - noflags_flags = ('--nobsdflags',) if location_config.get('flags') is False else () + noflags_flags = ('--nobsdflags',) if config.get('flags') is False else () if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version): - numeric_ids_flags = ('--numeric-ids',) if location_config.get('numeric_ids') else () + numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else () else: - numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_ids') else () + numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else () if feature.available(feature.Feature.UPLOAD_RATELIMIT, local_borg_version): upload_ratelimit_flags = ( @@ -415,7 +410,7 @@ def create_archive( ('--remote-ratelimit', str(upload_rate_limit)) if upload_rate_limit else () ) - if stream_processes and location_config.get('read_special') is False: + if stream_processes and config.get('read_special') is False: logger.warning( f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.' ) @@ -423,23 +418,19 @@ def create_archive( create_command = ( tuple(local_path.split(' ')) + ('create',) - + make_pattern_flags(location_config, pattern_file.name if pattern_file else None) - + make_exclude_flags(location_config, exclude_file.name if exclude_file else None) + + make_pattern_flags(config, pattern_file.name if pattern_file else None) + + make_exclude_flags(config, exclude_file.name if exclude_file else None) + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ()) + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ()) + (('--chunker-params', chunker_params) if chunker_params else ()) + (('--compression', compression) if compression else ()) + upload_ratelimit_flags - + ( - ('--one-file-system',) - if location_config.get('one_file_system') or stream_processes - else () - ) + + (('--one-file-system',) if config.get('one_file_system') or stream_processes else ()) + numeric_ids_flags + atime_flags - + (('--noctime',) if location_config.get('ctime') is False else ()) - + (('--nobirthtime',) if location_config.get('birthtime') is False else ()) - + (('--read-special',) if location_config.get('read_special') or stream_processes else ()) + + (('--noctime',) if config.get('ctime') is False else ()) + + (('--nobirthtime',) if config.get('birthtime') is False else ()) + + (('--read-special',) if config.get('read_special') or stream_processes else ()) + noflags_flags + (('--files-cache', files_cache) if files_cache else ()) + (('--remote-path', remote_path) if remote_path else ()) @@ -470,11 +461,11 @@ def create_archive( # the terminal directly. output_file = DO_NOT_CAPTURE if progress else None - borg_environment = environment.make_environment(storage_config) + borg_environment = environment.make_environment(config) # If database hooks are enabled (as indicated by streaming processes), exclude files that might # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True. - if stream_processes and not location_config.get('read_special'): + if stream_processes and not config.get('read_special'): logger.debug(f'{repository_path}: Collecting special file paths') special_file_paths = collect_special_file_paths( create_command, @@ -490,11 +481,11 @@ def create_archive( ) exclude_file = write_pattern_file( expand_home_directories( - tuple(location_config.get('exclude_patterns') or ()) + special_file_paths + tuple(config.get('exclude_patterns') or ()) + special_file_paths ), pattern_file=exclude_file, ) - create_command += make_exclude_flags(location_config, exclude_file.name) + create_command += make_exclude_flags(config, exclude_file.name) create_command += ( (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ()) diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 1b14369a..6ee3cbef 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -17,15 +17,15 @@ DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = { } -def make_environment(storage_config): +def make_environment(config): ''' - Given a borgmatic storage configuration dict, return its options converted to a Borg environment + Given a borgmatic configuration dict, return its options converted to a Borg environment variable dict. ''' environment = {} for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items(): - value = storage_config.get(option_name) + value = config.get(option_name) if value: environment[environment_variable_name] = str(value) @@ -34,7 +34,7 @@ def make_environment(storage_config): option_name, environment_variable_name, ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items(): - value = storage_config.get(option_name, False) + value = config.get(option_name, False) environment[environment_variable_name] = 'yes' if value else 'no' return environment diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index b6d9a04c..47e3c20d 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -13,7 +13,7 @@ def export_tar_archive( archive, paths, destination_path, - storage_config, + config, local_borg_version, global_arguments, local_path='borg', @@ -24,16 +24,16 @@ def export_tar_archive( ): ''' Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to - export from the archive, a destination path to export to, a storage configuration dict, the - local Borg version, optional local and remote Borg paths, an optional filter program, whether to - include per-file details, and an optional number of path components to strip, export the archive - into the given destination path as a tar-formatted file. + export from the archive, a destination path to export to, a configuration dict, the local Borg + version, optional local and remote Borg paths, an optional filter program, whether to include + per-file details, and an optional number of path components to strip, export the archive into + the given destination path as a tar-formatted file. If the destination path is "-", then stream the output to stdout instead of to a file. ''' borgmatic.logger.add_custom_log_levels() - umask = storage_config.get('umask', None) - lock_wait = storage_config.get('lock_wait', None) + umask = config.get('umask', None) + lock_wait = config.get('lock_wait', None) full_command = ( (local_path, 'export-tar') @@ -70,5 +70,5 @@ def export_tar_archive( output_file=DO_NOT_CAPTURE if destination_path == '-' else None, output_log_level=output_log_level, borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), + extra_environment=environment.make_environment(config), ) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index d5465bb9..dec203fc 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) def extract_last_archive_dry_run( - storage_config, + config, local_borg_version, global_arguments, repository_path, @@ -32,7 +32,7 @@ def extract_last_archive_dry_run( last_archive_name = rlist.resolve_archive_name( repository_path, 'latest', - storage_config, + config, local_borg_version, global_arguments, local_path, @@ -43,7 +43,7 @@ def extract_last_archive_dry_run( return list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else () - borg_environment = environment.make_environment(storage_config) + borg_environment = environment.make_environment(config) full_extract_command = ( (local_path, 'extract', '--dry-run') + (('--remote-path', remote_path) if remote_path else ()) @@ -66,8 +66,7 @@ def extract_archive( repository, archive, paths, - location_config, - storage_config, + config, local_borg_version, global_arguments, local_path='borg', @@ -80,22 +79,22 @@ def extract_archive( ''' Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to restore from the archive, the local Borg version string, an argparse.Namespace of global - arguments, location/storage configuration dicts, optional local and remote Borg paths, and an - optional destination path to extract to, extract the archive into the current directory. + arguments, a configuration dict, optional local and remote Borg paths, and an optional + destination path to extract to, extract the archive into the current directory. If extract to stdout is True, then start the extraction streaming to stdout, and return that extract process as an instance of subprocess.Popen. ''' - umask = storage_config.get('umask', None) - lock_wait = storage_config.get('lock_wait', None) + umask = config.get('umask', None) + lock_wait = config.get('lock_wait', None) if progress and extract_to_stdout: raise ValueError('progress and extract_to_stdout cannot both be set') if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version): - numeric_ids_flags = ('--numeric-ids',) if location_config.get('numeric_ids') else () + numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else () else: - numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_ids') else () + numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else () if strip_components == 'all': if not paths: @@ -127,7 +126,7 @@ def extract_archive( + (tuple(paths) if paths else ()) ) - borg_environment = environment.make_environment(storage_config) + borg_environment = environment.make_environment(config) # The progress output isn't compatible with captured and logged output, as progress messes with # the terminal directly. diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index 91520e00..9a8bdda2 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def display_archives_info( repository_path, - storage_config, + config, local_borg_version, info_arguments, global_arguments, @@ -17,12 +17,12 @@ def display_archives_info( remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, global + Given a local or remote repository path, a configuration dict, the local Borg version, global arguments as an argparse.Namespace, and the arguments to the info action, display summary information for Borg archives in the repository or return JSON summary information. ''' borgmatic.logger.add_custom_log_levels() - lock_wait = storage_config.get('lock_wait', None) + lock_wait = config.get('lock_wait', None) full_command = ( (local_path, 'info') @@ -50,8 +50,8 @@ def display_archives_info( flags.make_match_archives_flags( info_arguments.match_archives or info_arguments.archive - or storage_config.get('match_archives'), - storage_config.get('archive_name_format'), + or config.get('match_archives'), + config.get('archive_name_format'), local_borg_version, ) ) @@ -65,12 +65,12 @@ def display_archives_info( if info_arguments.json: return execute_command_and_capture_output( full_command, - extra_environment=environment.make_environment(storage_config), + extra_environment=environment.make_environment(config), ) else: execute_command( full_command, output_log_level=logging.ANSWER, borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), + extra_environment=environment.make_environment(config), ) diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 5ec1480d..b3db8e90 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -21,7 +21,7 @@ MAKE_FLAGS_EXCLUDES = ( def make_list_command( repository_path, - storage_config, + config, local_borg_version, list_arguments, global_arguments, @@ -29,11 +29,11 @@ def make_list_command( remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the arguments to the list - action, and local and remote Borg paths, return a command as a tuple to list archives or paths - within an archive. + Given a local or remote repository path, a configuration dict, the arguments to the list action, + and local and remote Borg paths, return a command as a tuple to list archives or paths within an + archive. ''' - lock_wait = storage_config.get('lock_wait', None) + lock_wait = config.get('lock_wait', None) return ( (local_path, 'list') @@ -89,7 +89,7 @@ def make_find_paths(find_paths): def capture_archive_listing( repository_path, archive, - storage_config, + config, local_borg_version, global_arguments, list_path=None, @@ -97,18 +97,18 @@ def capture_archive_listing( remote_path=None, ): ''' - Given a local or remote repository path, an archive name, a storage config dict, the local Borg + Given a local or remote repository path, an archive name, a configuration dict, the local Borg version, global arguments as an argparse.Namespace, the archive path in which to list files, and local and remote Borg paths, capture the output of listing that archive and return it as a list of file paths. ''' - borg_environment = environment.make_environment(storage_config) + borg_environment = environment.make_environment(config) return tuple( execute_command_and_capture_output( make_list_command( repository_path, - storage_config, + config, local_borg_version, argparse.Namespace( repository=repository_path, @@ -131,7 +131,7 @@ def capture_archive_listing( def list_archive( repository_path, - storage_config, + config, local_borg_version, list_arguments, global_arguments, @@ -139,7 +139,7 @@ def list_archive( remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, global + Given a local or remote repository path, a configuration dict, the local Borg version, global arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace, and local and remote Borg paths, display the output of listing the files of a Borg archive (or return JSON output). If list_arguments.find_paths are given, list the files by searching across @@ -167,7 +167,7 @@ def list_archive( ) return rlist.list_repository( repository_path, - storage_config, + config, local_borg_version, rlist_arguments, global_arguments, @@ -187,7 +187,7 @@ def list_archive( 'The --json flag on the list action is not supported when using the --archive/--find flags.' ) - borg_environment = environment.make_environment(storage_config) + borg_environment = environment.make_environment(config) # If there are any paths to find (and there's not a single archive already selected), start by # getting a list of archives to search. @@ -209,7 +209,7 @@ def list_archive( execute_command_and_capture_output( rlist.make_rlist_command( repository_path, - storage_config, + config, local_borg_version, rlist_arguments, global_arguments, @@ -238,7 +238,7 @@ def list_archive( main_command = make_list_command( repository_path, - storage_config, + config, local_borg_version, archive_arguments, global_arguments, diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 80cfa8d7..9d034688 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -10,7 +10,7 @@ def mount_archive( repository_path, archive, mount_arguments, - storage_config, + config, local_borg_version, global_arguments, local_path='borg', @@ -22,8 +22,8 @@ def mount_archive( dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional local and remote Borg paths, mount the archive onto the mount point. ''' - umask = storage_config.get('umask', None) - lock_wait = storage_config.get('lock_wait', None) + umask = config.get('umask', None) + lock_wait = config.get('lock_wait', None) full_command = ( (local_path, 'mount') @@ -58,7 +58,7 @@ def mount_archive( + (tuple(mount_arguments.paths) if mount_arguments.paths else ()) ) - borg_environment = environment.make_environment(storage_config) + borg_environment = environment.make_environment(config) # Don't capture the output when foreground mode is used so that ctrl-C can work properly. if mount_arguments.foreground: diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index a85cacf9..46c7f34b 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -7,9 +7,9 @@ from borgmatic.execute import execute_command logger = logging.getLogger(__name__) -def make_prune_flags(storage_config, retention_config, local_borg_version): +def make_prune_flags(config, local_borg_version): ''' - Given a retention config dict mapping from option name to value, transform it into an sequence of + Given a configuration dict mapping from option name to value, transform it into an sequence of command-line flags. For example, given a retention config of: @@ -23,12 +23,12 @@ def make_prune_flags(storage_config, retention_config, local_borg_version): ('--keep-monthly', '6'), ) ''' - config = retention_config.copy() - prefix = config.pop('prefix', None) - flag_pairs = ( - ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items() + ('--' + option_name.replace('_', '-'), str(value)) + for option_name, value in config.items() + if option_name.startswith('keep_') ) + prefix = config.get('prefix') return tuple(element for pair in flag_pairs for element in pair) + ( ( @@ -39,8 +39,8 @@ def make_prune_flags(storage_config, retention_config, local_borg_version): if prefix else ( flags.make_match_archives_flags( - storage_config.get('match_archives'), - storage_config.get('archive_name_format'), + config.get('match_archives'), + config.get('archive_name_format'), local_borg_version, ) ) @@ -50,8 +50,7 @@ def make_prune_flags(storage_config, retention_config, local_borg_version): def prune_archives( dry_run, repository_path, - storage_config, - retention_config, + config, local_borg_version, prune_arguments, global_arguments, @@ -59,18 +58,17 @@ def prune_archives( remote_path=None, ): ''' - Given dry-run flag, a local or remote repository path, a storage config dict, and a - retention config dict, prune Borg archives according to the retention policy specified in that - configuration. + Given dry-run flag, a local or remote repository path, and a configuration dict, prune Borg + archives according to the retention policy specified in that configuration. ''' borgmatic.logger.add_custom_log_levels() - umask = storage_config.get('umask', None) - lock_wait = storage_config.get('lock_wait', None) - extra_borg_options = storage_config.get('extra_borg_options', {}).get('prune', '') + umask = config.get('umask', None) + lock_wait = config.get('lock_wait', None) + extra_borg_options = config.get('extra_borg_options', {}).get('prune', '') full_command = ( (local_path, 'prune') - + make_prune_flags(storage_config, retention_config, local_borg_version) + + make_prune_flags(config, local_borg_version) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + (('--log-json',) if global_arguments.log_json else ()) @@ -97,5 +95,5 @@ def prune_archives( full_command, output_log_level=output_log_level, borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), + extra_environment=environment.make_environment(config), ) diff --git a/borgmatic/borg/rcreate.py b/borgmatic/borg/rcreate.py index 54a865c5..8fc70d95 100644 --- a/borgmatic/borg/rcreate.py +++ b/borgmatic/borg/rcreate.py @@ -14,7 +14,7 @@ RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2 def create_repository( dry_run, repository_path, - storage_config, + config, local_borg_version, global_arguments, encryption_mode, @@ -27,15 +27,15 @@ def create_repository( remote_path=None, ): ''' - Given a dry-run flag, a local or remote repository path, a storage configuration dict, the local - Borg version, a Borg encryption mode, the path to another repo whose key material should be - reused, whether the repository should be append-only, and the storage quota to use, create the + Given a dry-run flag, a local or remote repository path, a configuration dict, the local Borg + version, a Borg encryption mode, the path to another repo whose key material should be reused, + whether the repository should be append-only, and the storage quota to use, create the repository. If the repository already exists, then log and skip creation. ''' try: rinfo.display_repository_info( repository_path, - storage_config, + config, local_borg_version, argparse.Namespace(json=True), global_arguments, @@ -48,8 +48,8 @@ def create_repository( if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE: raise - lock_wait = storage_config.get('lock_wait') - extra_borg_options = storage_config.get('extra_borg_options', {}).get('rcreate', '') + lock_wait = config.get('lock_wait') + extra_borg_options = config.get('extra_borg_options', {}).get('rcreate', '') rcreate_command = ( (local_path,) @@ -82,5 +82,5 @@ def create_repository( rcreate_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), + extra_environment=environment.make_environment(config), ) diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index e1542d28..a7ae8229 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def display_repository_info( repository_path, - storage_config, + config, local_borg_version, rinfo_arguments, global_arguments, @@ -17,12 +17,12 @@ def display_repository_info( remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, the + Given a local or remote repository path, a configuration dict, the local Borg version, the arguments to the rinfo action, and global arguments as an argparse.Namespace, display summary information for the Borg repository or return JSON summary information. ''' borgmatic.logger.add_custom_log_levels() - lock_wait = storage_config.get('lock_wait', None) + lock_wait = config.get('lock_wait', None) full_command = ( (local_path,) @@ -48,7 +48,7 @@ def display_repository_info( + flags.make_repository_flags(repository_path, local_borg_version) ) - extra_environment = environment.make_environment(storage_config) + extra_environment = environment.make_environment(config) if rinfo_arguments.json: return execute_command_and_capture_output( diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index ba45aa0a..b532a6aa 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -10,14 +10,14 @@ logger = logging.getLogger(__name__) def resolve_archive_name( repository_path, archive, - storage_config, + config, local_borg_version, global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, an archive name, a storage config dict, the local Borg + Given a local or remote repository path, an archive name, a configuration dict, the local Borg version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path, return the archive name. But if the archive name is "latest", then instead introspect the repository for the latest archive and return its name. @@ -34,7 +34,7 @@ def resolve_archive_name( ) + flags.make_flags('remote-path', remote_path) + flags.make_flags('log-json', global_arguments.log_json) - + flags.make_flags('lock-wait', storage_config.get('lock_wait')) + + flags.make_flags('lock-wait', config.get('lock_wait')) + flags.make_flags('last', 1) + ('--short',) + flags.make_repository_flags(repository_path, local_borg_version) @@ -42,7 +42,7 @@ def resolve_archive_name( output = execute_command_and_capture_output( full_command, - extra_environment=environment.make_environment(storage_config), + extra_environment=environment.make_environment(config), ) try: latest_archive = output.strip().splitlines()[-1] @@ -59,7 +59,7 @@ MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives') def make_rlist_command( repository_path, - storage_config, + config, local_borg_version, rlist_arguments, global_arguments, @@ -67,7 +67,7 @@ def make_rlist_command( remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, the + Given a local or remote repository path, a configuration dict, the local Borg version, the arguments to the rlist action, global arguments as an argparse.Namespace instance, and local and remote Borg paths, return a command as a tuple to list archives with a repository. ''' @@ -88,7 +88,7 @@ def make_rlist_command( ) + flags.make_flags('remote-path', remote_path) + flags.make_flags('log-json', global_arguments.log_json) - + flags.make_flags('lock-wait', storage_config.get('lock_wait')) + + flags.make_flags('lock-wait', config.get('lock_wait')) + ( ( flags.make_flags('match-archives', f'sh:{rlist_arguments.prefix}*') @@ -98,8 +98,8 @@ def make_rlist_command( if rlist_arguments.prefix else ( flags.make_match_archives_flags( - rlist_arguments.match_archives or storage_config.get('match_archives'), - storage_config.get('archive_name_format'), + rlist_arguments.match_archives or config.get('match_archives'), + config.get('archive_name_format'), local_borg_version, ) ) @@ -111,7 +111,7 @@ def make_rlist_command( def list_repository( repository_path, - storage_config, + config, local_borg_version, rlist_arguments, global_arguments, @@ -119,17 +119,17 @@ def list_repository( remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, the + Given a local or remote repository path, a configuration dict, the local Borg version, the arguments to the list action, global arguments as an argparse.Namespace instance, and local and remote Borg paths, display the output of listing Borg archives in the given repository (or return JSON output). ''' borgmatic.logger.add_custom_log_levels() - borg_environment = environment.make_environment(storage_config) + borg_environment = environment.make_environment(config) main_command = make_rlist_command( repository_path, - storage_config, + config, local_borg_version, rlist_arguments, global_arguments, diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index d8f3978f..f91349fc 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) def transfer_archives( dry_run, repository_path, - storage_config, + config, local_borg_version, transfer_arguments, global_arguments, @@ -18,7 +18,7 @@ def transfer_archives( remote_path=None, ): ''' - Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg + Given a dry-run flag, a local or remote repository path, a configuration dict, the local Borg version, the arguments to the transfer action, and global arguments as an argparse.Namespace instance, transfer archives to the given repository. ''' @@ -30,7 +30,7 @@ def transfer_archives( + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + flags.make_flags('log-json', global_arguments.log_json) - + flags.make_flags('lock-wait', storage_config.get('lock_wait', None)) + + flags.make_flags('lock-wait', config.get('lock_wait', None)) + ( flags.make_flags_from_arguments( transfer_arguments, @@ -40,8 +40,8 @@ def transfer_archives( flags.make_match_archives_flags( transfer_arguments.match_archives or transfer_arguments.archive - or storage_config.get('match_archives'), - storage_config.get('archive_name_format'), + or config.get('match_archives'), + config.get('archive_name_format'), local_borg_version, ) ) @@ -56,5 +56,5 @@ def transfer_archives( output_log_level=logging.ANSWER, output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None, borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), + extra_environment=environment.make_environment(config), ) diff --git a/borgmatic/borg/version.py b/borgmatic/borg/version.py index d90a7aae..feb677ad 100644 --- a/borgmatic/borg/version.py +++ b/borgmatic/borg/version.py @@ -6,9 +6,9 @@ from borgmatic.execute import execute_command_and_capture_output logger = logging.getLogger(__name__) -def local_borg_version(storage_config, local_path='borg'): +def local_borg_version(config, local_path='borg'): ''' - Given a storage configuration dict and a local Borg binary path, return a version string for it. + Given a configuration dict and a local Borg binary path, return a version string for it. Raise OSError or CalledProcessError if there is a problem running Borg. Raise ValueError if the version cannot be parsed. @@ -20,7 +20,7 @@ def local_borg_version(storage_config, local_path='borg'): ) output = execute_command_and_capture_output( full_command, - extra_environment=environment.make_environment(storage_config), + extra_environment=environment.make_environment(config), ) try: diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 588e31c9..ce246e14 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -330,7 +330,7 @@ def make_parsers(): ) global_group.add_argument( '--override', - metavar='SECTION.OPTION=VALUE', + metavar='OPTION.SUBOPTION=VALUE', nargs='+', dest='overrides', action='extend', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 6aaad9d1..a869594e 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -58,16 +58,12 @@ def run_configuration(config_filename, config, arguments): * JSON output strings from successfully executing any actions that produce JSON * logging.LogRecord instances containing errors from any actions or backup hooks that fail ''' - (location, storage, retention, consistency, hooks) = ( - config.get(section_name, {}) - for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks') - ) global_arguments = arguments['global'] - local_path = location.get('local_path', 'borg') - remote_path = location.get('remote_path') - retries = storage.get('retries', 0) - retry_wait = storage.get('retry_wait', 0) + local_path = config.get('local_path', 'borg') + remote_path = config.get('remote_path') + retries = config.get('retries', 0) + retry_wait = config.get('retry_wait', 0) encountered_error = None error_repository = '' using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) @@ -75,7 +71,7 @@ def run_configuration(config_filename, config, arguments): monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED try: - local_borg_version = borg_version.local_borg_version(storage, local_path) + local_borg_version = borg_version.local_borg_version(config, local_path) except (OSError, CalledProcessError, ValueError) as error: yield from log_error_records(f'{config_filename}: Error getting local Borg version', error) return @@ -84,7 +80,7 @@ def run_configuration(config_filename, config, arguments): if monitoring_hooks_are_activated: dispatch.call_hooks( 'initialize_monitor', - hooks, + config, config_filename, monitor.MONITOR_HOOK_NAMES, monitoring_log_level, @@ -93,7 +89,7 @@ def run_configuration(config_filename, config, arguments): dispatch.call_hooks( 'ping_monitor', - hooks, + config, config_filename, monitor.MONITOR_HOOK_NAMES, monitor.State.START, @@ -109,7 +105,7 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: repo_queue = Queue() - for repo in location['repositories']: + for repo in config['repositories']: repo_queue.put( (repo, 0), ) @@ -129,11 +125,7 @@ def run_configuration(config_filename, config, arguments): yield from run_actions( arguments=arguments, config_filename=config_filename, - location=location, - storage=storage, - retention=retention, - consistency=consistency, - hooks=hooks, + config=config, local_path=local_path, remote_path=remote_path, local_borg_version=local_borg_version, @@ -172,7 +164,7 @@ def run_configuration(config_filename, config, arguments): # send logs irrespective of error dispatch.call_hooks( 'ping_monitor', - hooks, + config, config_filename, monitor.MONITOR_HOOK_NAMES, monitor.State.LOG, @@ -191,7 +183,7 @@ def run_configuration(config_filename, config, arguments): if monitoring_hooks_are_activated: dispatch.call_hooks( 'ping_monitor', - hooks, + config, config_filename, monitor.MONITOR_HOOK_NAMES, monitor.State.FINISH, @@ -200,7 +192,7 @@ def run_configuration(config_filename, config, arguments): ) dispatch.call_hooks( 'destroy_monitor', - hooks, + config, config_filename, monitor.MONITOR_HOOK_NAMES, monitoring_log_level, @@ -216,8 +208,8 @@ def run_configuration(config_filename, config, arguments): if encountered_error and using_primary_action: try: command.execute_hook( - hooks.get('on_error'), - hooks.get('umask'), + config.get('on_error'), + config.get('umask'), config_filename, 'on-error', global_arguments.dry_run, @@ -227,7 +219,7 @@ def run_configuration(config_filename, config, arguments): ) dispatch.call_hooks( 'ping_monitor', - hooks, + config, config_filename, monitor.MONITOR_HOOK_NAMES, monitor.State.FAIL, @@ -236,7 +228,7 @@ def run_configuration(config_filename, config, arguments): ) dispatch.call_hooks( 'destroy_monitor', - hooks, + config, config_filename, monitor.MONITOR_HOOK_NAMES, monitoring_log_level, @@ -253,11 +245,7 @@ def run_actions( *, arguments, config_filename, - location, - storage, - retention, - consistency, - hooks, + config, local_path, remote_path, local_borg_version, @@ -282,13 +270,13 @@ def run_actions( hook_context = { 'repository': repository_path, # Deprecated: For backwards compatibility with borgmatic < 1.6.0. - 'repositories': ','.join([repo['path'] for repo in location['repositories']]), + 'repositories': ','.join([repo['path'] for repo in config['repositories']]), 'log_file': global_arguments.log_file if global_arguments.log_file else '', } command.execute_hook( - hooks.get('before_actions'), - hooks.get('umask'), + config.get('before_actions'), + config.get('umask'), config_filename, 'pre-actions', global_arguments.dry_run, @@ -299,7 +287,7 @@ def run_actions( if action_name == 'rcreate': borgmatic.actions.rcreate.run_rcreate( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -309,7 +297,7 @@ def run_actions( elif action_name == 'transfer': borgmatic.actions.transfer.run_transfer( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -320,9 +308,7 @@ def run_actions( yield from borgmatic.actions.create.run_create( config_filename, repository, - location, - storage, - hooks, + config, hook_context, local_borg_version, action_arguments, @@ -335,9 +321,7 @@ def run_actions( borgmatic.actions.prune.run_prune( config_filename, repository, - storage, - retention, - hooks, + config, hook_context, local_borg_version, action_arguments, @@ -350,9 +334,7 @@ def run_actions( borgmatic.actions.compact.run_compact( config_filename, repository, - storage, - retention, - hooks, + config, hook_context, local_borg_version, action_arguments, @@ -362,14 +344,11 @@ def run_actions( remote_path, ) elif action_name == 'check': - if checks.repository_enabled_for_checks(repository, consistency): + if checks.repository_enabled_for_checks(repository, config): borgmatic.actions.check.run_check( config_filename, repository, - location, - storage, - consistency, - hooks, + config, hook_context, local_borg_version, action_arguments, @@ -381,9 +360,7 @@ def run_actions( borgmatic.actions.extract.run_extract( config_filename, repository, - location, - storage, - hooks, + config, hook_context, local_borg_version, action_arguments, @@ -394,7 +371,7 @@ def run_actions( elif action_name == 'export-tar': borgmatic.actions.export_tar.run_export_tar( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -404,7 +381,7 @@ def run_actions( elif action_name == 'mount': borgmatic.actions.mount.run_mount( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -414,9 +391,7 @@ def run_actions( elif action_name == 'restore': borgmatic.actions.restore.run_restore( repository, - location, - storage, - hooks, + config, local_borg_version, action_arguments, global_arguments, @@ -426,7 +401,7 @@ def run_actions( elif action_name == 'rlist': yield from borgmatic.actions.rlist.run_rlist( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -436,7 +411,7 @@ def run_actions( elif action_name == 'list': yield from borgmatic.actions.list.run_list( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -446,7 +421,7 @@ def run_actions( elif action_name == 'rinfo': yield from borgmatic.actions.rinfo.run_rinfo( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -456,7 +431,7 @@ def run_actions( elif action_name == 'info': yield from borgmatic.actions.info.run_info( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -466,7 +441,7 @@ def run_actions( elif action_name == 'break-lock': borgmatic.actions.break_lock.run_break_lock( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -476,7 +451,7 @@ def run_actions( elif action_name == 'borg': borgmatic.actions.borg.run_borg( repository, - storage, + config, local_borg_version, action_arguments, global_arguments, @@ -485,8 +460,8 @@ def run_actions( ) command.execute_hook( - hooks.get('after_actions'), - hooks.get('umask'), + config.get('after_actions'), + config.get('umask'), config_filename, 'post-actions', global_arguments.dry_run, @@ -613,7 +588,7 @@ def get_local_path(configs): Arbitrarily return the local path from the first configuration dict. Default to "borg" if not set. ''' - return next(iter(configs.values())).get('location', {}).get('local_path', 'borg') + return next(iter(configs.values())).get('local_path', 'borg') def collect_highlander_action_summary_logs(configs, arguments, configuration_parse_errors): @@ -627,6 +602,8 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par A highlander action is an action that cannot coexist with other actions on the borgmatic command-line, and borgmatic exits after processing such an action. ''' + add_custom_log_levels() + if 'bootstrap' in arguments: try: # No configuration file is needed for bootstrap. @@ -744,10 +721,9 @@ def collect_configuration_run_summary_logs(configs, arguments): if 'create' in arguments: try: for config_filename, config in configs.items(): - hooks = config.get('hooks', {}) command.execute_hook( - hooks.get('before_everything'), - hooks.get('umask'), + config.get('before_everything'), + config.get('umask'), config_filename, 'pre-everything', arguments['global'].dry_run, @@ -792,10 +768,9 @@ def collect_configuration_run_summary_logs(configs, arguments): if 'create' in arguments: try: for config_filename, config in configs.items(): - hooks = config.get('hooks', {}) command.execute_hook( - hooks.get('after_everything'), - hooks.get('umask'), + config.get('after_everything'), + config.get('umask'), config_filename, 'post-everything', arguments['global'].dry_run, diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 6ef8e3ae..01096547 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -11,7 +11,7 @@ INDENT = 4 SEQUENCE_INDENT = 2 -def _insert_newline_before_comment(config, field_name): +def insert_newline_before_comment(config, field_name): ''' Using some ruamel.yaml black magic, insert a blank line in the config right before the given field and its comments. @@ -21,10 +21,10 @@ def _insert_newline_before_comment(config, field_name): ) -def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): +def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): ''' Given a loaded configuration schema, generate and return sample config for it. Include comments - for each section based on the schema "description". + for each option based on the schema "description". ''' schema_type = schema.get('type') example = schema.get('example') @@ -33,13 +33,13 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): if schema_type == 'array': config = yaml.comments.CommentedSeq( - [_schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)] + [schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)] ) add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT)) elif schema_type == 'object': config = yaml.comments.CommentedMap( [ - (field_name, _schema_to_sample_configuration(sub_schema, level + 1)) + (field_name, schema_to_sample_configuration(sub_schema, level + 1)) for field_name, sub_schema in schema['properties'].items() ] ) @@ -53,13 +53,13 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): return config -def _comment_out_line(line): +def comment_out_line(line): # If it's already is commented out (or empty), there's nothing further to do! stripped_line = line.lstrip() if not stripped_line or stripped_line.startswith('#'): return line - # Comment out the names of optional sections, inserting the '#' after any indent for aesthetics. + # Comment out the names of optional options, inserting the '#' after any indent for aesthetics. matches = re.match(r'(\s*)', line) indent_spaces = matches.group(0) if matches else '' count_indent_spaces = len(indent_spaces) @@ -67,7 +67,7 @@ def _comment_out_line(line): return '# '.join((indent_spaces, line[count_indent_spaces:])) -def _comment_out_optional_configuration(rendered_config): +def comment_out_optional_configuration(rendered_config): ''' Post-process a rendered configuration string to comment out optional key/values, as determined by a sentinel in the comment before each key. @@ -92,7 +92,7 @@ def _comment_out_optional_configuration(rendered_config): if not line.strip(): optional = False - lines.append(_comment_out_line(line) if optional else line) + lines.append(comment_out_line(line) if optional else line) return '\n'.join(lines) @@ -165,7 +165,6 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): return -REQUIRED_SECTION_NAMES = {'location', 'retention'} REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'} COMMENTED_OUT_SENTINEL = 'COMMENT_OUT' @@ -185,7 +184,7 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa # If this is an optional key, add an indicator to the comment flagging it to be commented # out from the sample configuration. This sentinel is consumed by downstream processing that # does the actual commenting out. - if field_name not in REQUIRED_SECTION_NAMES and field_name not in REQUIRED_KEYS: + if field_name not in REQUIRED_KEYS: description = ( '\n'.join((description, COMMENTED_OUT_SENTINEL)) if description @@ -199,7 +198,7 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent) if index > 0: - _insert_newline_before_comment(config, field_name) + insert_newline_before_comment(config, field_name) RUAMEL_YAML_COMMENTS_INDEX = 1 @@ -284,7 +283,7 @@ def generate_sample_configuration( normalize.normalize(source_filename, source_config) destination_config = merge_source_configuration_into_destination( - _schema_to_sample_configuration(schema), source_config + schema_to_sample_configuration(schema), source_config ) if dry_run: @@ -292,6 +291,6 @@ def generate_sample_configuration( write_configuration( destination_filename, - _comment_out_optional_configuration(render_configuration(destination_config)), + comment_out_optional_configuration(render_configuration(destination_config)), overwrite=overwrite, ) diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index f5d071c9..e0fabfa6 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -97,8 +97,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor): ``` These includes are deep merged into the current configuration file. For instance, in this - example, any "retention" options in common.yaml will get merged into the "retention" section - in the example configuration file. + example, any "option" with sub-options in common.yaml will get merged into the corresponding + "option" with sub-options in the example configuration file. ''' representer = ruamel.yaml.representer.SafeRepresenter() @@ -116,7 +116,7 @@ def load_configuration(filename): ''' Load the given configuration file and return its contents as a data structure of nested dicts and lists. Also, replace any "{constant}" strings with the value of the "constant" key in the - "constants" section of the configuration file. + "constants" option of the configuration file. Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError if there are too many recursive includes. @@ -223,8 +223,8 @@ def deep_merge_nodes(nodes): If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged. The purpose of deep merging like this is to support, for instance, merging one borgmatic - configuration file into another for reuse, such that a configuration section ("retention", - etc.) does not completely replace the corresponding section in a merged file. + configuration file into another for reuse, such that a configuration option with sub-options + does not completely replace the corresponding option in a merged file. Raise ValueError if a merge is implied using two incompatible types. ''' diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index daadfeb4..83f2a20b 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -2,21 +2,70 @@ import logging import os +def normalize_sections(config_filename, config): + ''' + Given a configuration filename and a configuration dict of its loaded contents, airlift any + options out of sections ("location:", etc.) to the global scope and delete those sections. + Return any log message warnings produced based on the normalization performed. + + Raise ValueError if the "prefix" option is set in both "location" and "consistency" sections. + ''' + location = config.get('location') or {} + storage = config.get('storage') or {} + consistency = config.get('consistency') or {} + hooks = config.get('hooks') or {} + + if ( + location.get('prefix') + and consistency.get('prefix') + and location.get('prefix') != consistency.get('prefix') + ): + raise ValueError( + 'The retention prefix and the consistency prefix cannot have different values (unless one is not set).' + ) + + if storage.get('umask') and hooks.get('umask') and storage.get('umask') != hooks.get('umask'): + raise ValueError( + 'The storage umask and the hooks umask cannot have different values (unless one is not set).' + ) + + any_section_upgraded = False + + # Move any options from deprecated sections into the global scope. + for section_name in ('location', 'storage', 'retention', 'consistency', 'output', 'hooks'): + section_config = config.get(section_name) + + if section_config: + any_section_upgraded = True + del config[section_name] + config.update(section_config) + + if any_section_upgraded: + return [ + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: Configuration sections like location: and storage: are deprecated and support will be removed from a future release. Move all of your options out of sections to the global scope.', + ) + ) + ] + + return [] + + def normalize(config_filename, config): ''' Given a configuration filename and a configuration dict of its loaded contents, apply particular hard-coded rules to normalize the configuration to adhere to the current schema. Return any log message warnings produced based on the normalization performed. + + Raise ValueError the configuration cannot be normalized. ''' - logs = [] - location = config.get('location') or {} - storage = config.get('storage') or {} - consistency = config.get('consistency') or {} - retention = config.get('retention') or {} - hooks = config.get('hooks') or {} + logs = normalize_sections(config_filename, config) # Upgrade exclude_if_present from a string to a list. - exclude_if_present = location.get('exclude_if_present') + exclude_if_present = config.get('exclude_if_present') if isinstance(exclude_if_present, str): logs.append( logging.makeLogRecord( @@ -27,10 +76,10 @@ def normalize(config_filename, config): ) ) ) - config['location']['exclude_if_present'] = [exclude_if_present] + config['exclude_if_present'] = [exclude_if_present] # Upgrade various monitoring hooks from a string to a dict. - healthchecks = hooks.get('healthchecks') + healthchecks = config.get('healthchecks') if isinstance(healthchecks, str): logs.append( logging.makeLogRecord( @@ -41,9 +90,9 @@ def normalize(config_filename, config): ) ) ) - config['hooks']['healthchecks'] = {'ping_url': healthchecks} + config['healthchecks'] = {'ping_url': healthchecks} - cronitor = hooks.get('cronitor') + cronitor = config.get('cronitor') if isinstance(cronitor, str): logs.append( logging.makeLogRecord( @@ -54,9 +103,9 @@ def normalize(config_filename, config): ) ) ) - config['hooks']['cronitor'] = {'ping_url': cronitor} + config['cronitor'] = {'ping_url': cronitor} - pagerduty = hooks.get('pagerduty') + pagerduty = config.get('pagerduty') if isinstance(pagerduty, str): logs.append( logging.makeLogRecord( @@ -67,9 +116,9 @@ def normalize(config_filename, config): ) ) ) - config['hooks']['pagerduty'] = {'integration_key': pagerduty} + config['pagerduty'] = {'integration_key': pagerduty} - cronhub = hooks.get('cronhub') + cronhub = config.get('cronhub') if isinstance(cronhub, str): logs.append( logging.makeLogRecord( @@ -80,10 +129,10 @@ def normalize(config_filename, config): ) ) ) - config['hooks']['cronhub'] = {'ping_url': cronhub} + config['cronhub'] = {'ping_url': cronhub} # Upgrade consistency checks from a list of strings to a list of dicts. - checks = consistency.get('checks') + checks = config.get('checks') if isinstance(checks, list) and len(checks) and isinstance(checks[0], str): logs.append( logging.makeLogRecord( @@ -94,10 +143,10 @@ def normalize(config_filename, config): ) ) ) - config['consistency']['checks'] = [{'name': check_type} for check_type in checks] + config['checks'] = [{'name': check_type} for check_type in checks] # Rename various configuration options. - numeric_owner = location.pop('numeric_owner', None) + numeric_owner = config.pop('numeric_owner', None) if numeric_owner is not None: logs.append( logging.makeLogRecord( @@ -108,9 +157,9 @@ def normalize(config_filename, config): ) ) ) - config['location']['numeric_ids'] = numeric_owner + config['numeric_ids'] = numeric_owner - bsd_flags = location.pop('bsd_flags', None) + bsd_flags = config.pop('bsd_flags', None) if bsd_flags is not None: logs.append( logging.makeLogRecord( @@ -121,9 +170,9 @@ def normalize(config_filename, config): ) ) ) - config['location']['flags'] = bsd_flags + config['flags'] = bsd_flags - remote_rate_limit = storage.pop('remote_rate_limit', None) + remote_rate_limit = config.pop('remote_rate_limit', None) if remote_rate_limit is not None: logs.append( logging.makeLogRecord( @@ -134,10 +183,10 @@ def normalize(config_filename, config): ) ) ) - config['storage']['upload_rate_limit'] = remote_rate_limit + config['upload_rate_limit'] = remote_rate_limit # Upgrade remote repositories to ssh:// syntax, required in Borg 2. - repositories = location.get('repositories') + repositories = config.get('repositories') if repositories: if isinstance(repositories[0], str): logs.append( @@ -149,11 +198,11 @@ def normalize(config_filename, config): ) ) ) - config['location']['repositories'] = [ - {'path': repository} for repository in repositories - ] - repositories = config['location']['repositories'] - config['location']['repositories'] = [] + config['repositories'] = [{'path': repository} for repository in repositories] + repositories = config['repositories'] + + config['repositories'] = [] + for repository_dict in repositories: repository_path = repository_dict['path'] if '~' in repository_path: @@ -171,14 +220,14 @@ def normalize(config_filename, config): updated_repository_path = os.path.abspath( repository_path.partition('file://')[-1] ) - config['location']['repositories'].append( + config['repositories'].append( dict( repository_dict, path=updated_repository_path, ) ) elif repository_path.startswith('ssh://'): - config['location']['repositories'].append(repository_dict) + config['repositories'].append(repository_dict) else: rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" logs.append( @@ -190,16 +239,16 @@ def normalize(config_filename, config): ) ) ) - config['location']['repositories'].append( + config['repositories'].append( dict( repository_dict, path=rewritten_repository_path, ) ) else: - config['location']['repositories'].append(repository_dict) + config['repositories'].append(repository_dict) - if consistency.get('prefix') or retention.get('prefix'): + if config.get('prefix'): logs.append( logging.makeLogRecord( dict( diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index aacf375d..05173d2c 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -32,19 +32,33 @@ def convert_value_type(value): return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) +LEGACY_SECTION_NAMES = {'location', 'storage', 'retention', 'consistency', 'output', 'hooks'} + + +def strip_section_names(parsed_override_key): + ''' + Given a parsed override key as a tuple of option and suboption names, strip out any initial + legacy section names, since configuration file normalization also strips them out. + ''' + if parsed_override_key[0] in LEGACY_SECTION_NAMES: + return parsed_override_key[1:] + + return parsed_override_key + + def parse_overrides(raw_overrides): ''' - Given a sequence of configuration file override strings in the form of "section.option=value", + Given a sequence of configuration file override strings in the form of "option.suboption=value", parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For instance, given the following raw overrides: - ['section.my_option=value1', 'section.other_option=value2'] + ['my_option.suboption=value1', 'other_option=value2'] ... return this: ( - (('section', 'my_option'), 'value1'), - (('section', 'other_option'), 'value2'), + (('my_option', 'suboption'), 'value1'), + (('other_option'), 'value2'), ) Raise ValueError if an override can't be parsed. @@ -59,13 +73,13 @@ def parse_overrides(raw_overrides): raw_keys, value = raw_override.split('=', 1) parsed_overrides.append( ( - tuple(raw_keys.split('.')), + strip_section_names(tuple(raw_keys.split('.'))), convert_value_type(value), ) ) except ValueError: raise ValueError( - f"Invalid override '{raw_override}'. Make sure you use the form: SECTION.OPTION=VALUE" + f"Invalid override '{raw_override}'. Make sure you use the form: OPTION=VALUE or OPTION.SUBOPTION=VALUE" ) except ruamel.yaml.error.YAMLError as error: raise ValueError(f"Invalid override '{raw_override}': {error.problem}") @@ -76,7 +90,7 @@ def parse_overrides(raw_overrides): def apply_overrides(config, raw_overrides): ''' Given a configuration dict and a sequence of configuration file override strings in the form of - "section.option=value", parse each override and set it the configuration dict. + "option.suboption=value", parse each override and set it the configuration dict. ''' overrides = parse_overrides(raw_overrides) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3f1b3cb4..b0cbe6ae 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1,6 +1,6 @@ type: object required: - - location + - repositories additionalProperties: false properties: constants: @@ -14,1391 +14,1268 @@ properties: example: hostname: myhostname prefix: myprefix - location: - type: object + source_directories: + type: array + items: + type: string description: | - Where to look for files to backup, and where to store those backups. - See https://borgbackup.readthedocs.io/en/stable/quickstart.html and - https://borgbackup.readthedocs.io/en/stable/usage/create.html - for details. - required: - - repositories + List of source directories and files to backup. Globs and tildes are + expanded. Do not backslash spaces in path names. + example: + - /home + - /etc + - /var/log/syslog* + - /home/user/path with spaces + repositories: + type: array + items: + type: object + required: + - path + properties: + path: + type: string + example: ssh://user@backupserver/./{fqdn} + label: + type: string + example: backupserver + description: | + A required list of local or remote repositories with paths and + optional labels (which can be used with the --repository flag to + select a repository). Tildes are expanded. Multiple repositories are + backed up to in sequence. Borg placeholders can be used. See the + output of "borg help placeholders" for details. See ssh_command for + SSH options like identity file or port. If systemd service is used, + then add local repository paths in the systemd service file to the + ReadWritePaths list. Prior to borgmatic 1.7.10, repositories was a + list of plain path strings. + example: + - path: ssh://user@backupserver/./sourcehostname.borg + label: backupserver + - path: /mnt/backup + label: local + working_directory: + type: string + description: | + Working directory for the "borg create" command. Tildes are + expanded. Useful for backing up using relative paths. See + http://borgbackup.readthedocs.io/en/stable/usage/create.html for + details. Defaults to not set. + example: /path/to/working/directory + one_file_system: + type: boolean + description: | + Stay in same file system; do not cross mount points beyond the given + source directories. Defaults to false. But when a database hook is + used, the setting here is ignored and one_file_system is considered + true. + example: true + numeric_ids: + type: boolean + description: | + Only store/extract numeric user and group identifiers. Defaults to + false. + example: true + atime: + type: boolean + description: | + Store atime into archive. Defaults to true in Borg < 1.2, false in + Borg 1.2+. + example: false + ctime: + type: boolean + description: Store ctime into archive. Defaults to true. + example: false + birthtime: + type: boolean + description: | + Store birthtime (creation date) into archive. Defaults to true. + example: false + read_special: + type: boolean + description: | + Use Borg's --read-special flag to allow backup of block and other + special devices. Use with caution, as it will lead to problems if + used when backing up special devices such as /dev/zero. Defaults to + false. But when a database hook is used, the setting here is ignored + and read_special is considered true. + example: false + flags: + type: boolean + description: | + Record filesystem flags (e.g. NODUMP, IMMUTABLE) in archive. + Defaults to true. + example: true + files_cache: + type: string + description: | + Mode in which to operate the files cache. See + http://borgbackup.readthedocs.io/en/stable/usage/create.html for + details. Defaults to "ctime,size,inode". + example: ctime,size,inode + local_path: + type: string + description: | + Alternate Borg local executable. Defaults to "borg". + example: borg1 + remote_path: + type: string + description: | + Alternate Borg remote executable. Defaults to "borg". + example: borg1 + patterns: + type: array + items: + type: string + description: | + Any paths matching these patterns are included/excluded from + backups. Globs are expanded. (Tildes are not.) See the output of + "borg help patterns" for more details. Quote any value if it + contains leading punctuation, so it parses correctly. Note that only + one of "patterns" and "source_directories" may be used. + example: + - 'R /' + - '- /home/*/.cache' + - '+ /home/susan' + - '- /home/*' + patterns_from: + type: array + items: + type: string + description: | + Read include/exclude patterns from one or more separate named files, + one pattern per line. Note that Borg considers this option + experimental. See the output of "borg help patterns" for more + details. + example: + - /etc/borgmatic/patterns + exclude_patterns: + type: array + items: + type: string + description: | + Any paths matching these patterns are excluded from backups. Globs + and tildes are expanded. Note that a glob pattern must either start + with a glob or be an absolute path. Do not backslash spaces in path + names. See the output of "borg help patterns" for more details. + example: + - '*.pyc' + - /home/*/.cache + - '*/.vim*.tmp' + - /etc/ssl + - /home/user/path with spaces + exclude_from: + type: array + items: + type: string + description: | + Read exclude patterns from one or more separate named files, one + pattern per line. See the output of "borg help patterns" for more + details. + example: + - /etc/borgmatic/excludes + exclude_caches: + type: boolean + description: | + Exclude directories that contain a CACHEDIR.TAG file. See + http://www.brynosaurus.com/cachedir/spec.html for details. Defaults + to false. + example: true + exclude_if_present: + type: array + items: + type: string + description: | + Exclude directories that contain a file with the given filenames. + Defaults to not set. + example: + - .nobackup + keep_exclude_tags: + type: boolean + description: | + If true, the exclude_if_present filename is included in backups. + Defaults to false, meaning that the exclude_if_present filename is + omitted from backups. + example: true + exclude_nodump: + type: boolean + description: | + Exclude files with the NODUMP flag. Defaults to false. + example: true + borgmatic_source_directory: + type: string + description: | + Path for additional source files used for temporary internal state + like borgmatic database dumps. Note that changing this path prevents + "borgmatic restore" from finding any database dumps created before + the change. Defaults to ~/.borgmatic + example: /tmp/borgmatic + source_directories_must_exist: + type: boolean + description: | + If true, then source directories must exist, otherwise an error is + raised. Defaults to false. + example: true + encryption_passcommand: + type: string + description: | + The standard output of this command is used to unlock the encryption + key. Only use on repositories that were initialized with + passcommand/repokey/keyfile encryption. Note that if both + encryption_passcommand and encryption_passphrase are set, then + encryption_passphrase takes precedence. Defaults to not set. + example: "secret-tool lookup borg-repository repo-name" + encryption_passphrase: + type: string + description: | + Passphrase to unlock the encryption key with. Only use on + repositories that were initialized with passphrase/repokey/keyfile + encryption. Quote the value if it contains punctuation, so it parses + correctly. And backslash any quote or backslash literals as well. + Defaults to not set. + example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" + checkpoint_interval: + type: integer + description: | + Number of seconds between each checkpoint during a long-running + backup. See https://borgbackup.readthedocs.io/en/stable/faq.html for + details. Defaults to checkpoints every 1800 seconds (30 minutes). + example: 1800 + checkpoint_volume: + type: integer + description: | + Number of backed up bytes between each checkpoint during a + long-running backup. Only supported with Borg 2+. See + https://borgbackup.readthedocs.io/en/stable/faq.html for details. + Defaults to only time-based checkpointing (see + "checkpoint_interval") instead of volume-based checkpointing. + example: 1048576 + chunker_params: + type: string + description: | + Specify the parameters passed to then chunker (CHUNK_MIN_EXP, + CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). See + https://borgbackup.readthedocs.io/en/stable/internals.html for + details. Defaults to "19,23,21,4095". + example: 19,23,21,4095 + compression: + type: string + description: | + Type of compression to use when creating archives. See + http://borgbackup.readthedocs.io/en/stable/usage/create.html for + details. Defaults to "lz4". + example: lz4 + upload_rate_limit: + type: integer + description: | + Remote network upload rate limit in kiBytes/second. Defaults to + unlimited. + example: 100 + retries: + type: integer + description: | + Number of times to retry a failing backup before giving up. Defaults + to 0 (i.e., does not attempt retry). + example: 3 + retry_wait: + type: integer + description: | + Wait time between retries (in seconds) to allow transient issues to + pass. Increases after each retry as a form of backoff. Defaults to 0 + (no wait). + example: 10 + temporary_directory: + type: string + description: | + Directory where temporary files are stored. Defaults to $TMPDIR. + example: /path/to/tmpdir + ssh_command: + type: string + description: | + Command to use instead of "ssh". This can be used to specify ssh + options. Defaults to not set. + example: ssh -i /path/to/private/key + borg_base_directory: + type: string + description: | + Base path used for various Borg directories. Defaults to $HOME, + ~$USER, or ~. + example: /path/to/base + borg_config_directory: + type: string + description: | + Path for Borg configuration files. Defaults to + $borg_base_directory/.config/borg + example: /path/to/base/config + borg_cache_directory: + type: string + description: | + Path for Borg cache files. Defaults to + $borg_base_directory/.cache/borg + example: /path/to/base/cache + borg_files_cache_ttl: + type: integer + description: | + Maximum time to live (ttl) for entries in the Borg files cache. + example: 20 + borg_security_directory: + type: string + description: | + Path for Borg security and encryption nonce files. Defaults to + $borg_base_directory/.config/borg/security + example: /path/to/base/config/security + borg_keys_directory: + type: string + description: | + Path for Borg encryption key files. Defaults to + $borg_base_directory/.config/borg/keys + example: /path/to/base/config/keys + umask: + type: integer + description: | + Umask used for when executing Borg or calling hooks. Defaults to + 0077 for Borg or the umask that borgmatic is run with for hooks. + example: 0077 + lock_wait: + type: integer + description: | + Maximum seconds to wait for acquiring a repository/cache lock. + Defaults to 1. + example: 5 + archive_name_format: + type: string + description: | + Name of the archive. Borg placeholders can be used. See the output + of "borg help placeholders" for details. Defaults to + "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". When running actions like + rlist, info, or check, borgmatic automatically tries to match only + archives created with this name format. + example: "{hostname}-documents-{now}" + match_archives: + type: string + description: | + A Borg pattern for filtering down the archives used by borgmatic + actions that operate on multiple archives. For Borg 1.x, use a shell + pattern here and see the output of "borg help placeholders" for + details. For Borg 2.x, see the output of "borg help match-archives". + If match_archives is not specified, borgmatic defaults to deriving + the match_archives value from archive_name_format. + example: "sh:{hostname}-*" + relocated_repo_access_is_ok: + type: boolean + description: | + Bypass Borg error about a repository that has been moved. Defaults + to false. + example: true + unknown_unencrypted_repo_access_is_ok: + type: boolean + description: | + Bypass Borg error about a previously unknown unencrypted repository. + Defaults to false. + example: true + extra_borg_options: + type: object additionalProperties: false properties: - source_directories: + init: + type: string + description: | + Extra command-line options to pass to "borg init". + example: "--extra-option" + create: + type: string + description: | + Extra command-line options to pass to "borg create". + example: "--extra-option" + prune: + type: string + description: | + Extra command-line options to pass to "borg prune". + example: "--extra-option" + compact: + type: string + description: | + Extra command-line options to pass to "borg compact". + example: "--extra-option" + check: + type: string + description: | + Extra command-line options to pass to "borg check". + example: "--extra-option" + description: | + Additional options to pass directly to particular Borg commands, + handy for Borg options that borgmatic does not yet support natively. + Note that borgmatic does not perform any validation on these + options. Running borgmatic with "--verbosity 2" shows the exact Borg + command-line invocation. + keep_within: + type: string + description: Keep all archives within this time interval. + example: 3H + keep_secondly: + type: integer + description: Number of secondly archives to keep. + example: 60 + keep_minutely: + type: integer + description: Number of minutely archives to keep. + example: 60 + keep_hourly: + type: integer + description: Number of hourly archives to keep. + example: 24 + keep_daily: + type: integer + description: Number of daily archives to keep. + example: 7 + keep_weekly: + type: integer + description: Number of weekly archives to keep. + example: 4 + keep_monthly: + type: integer + description: Number of monthly archives to keep. + example: 6 + keep_yearly: + type: integer + description: Number of yearly archives to keep. + example: 1 + prefix: + type: string + description: | + Deprecated. When pruning or checking archives, only consider archive + names starting with this prefix. Borg placeholders can be used. See + the output of "borg help placeholders" for details. If a prefix is + not specified, borgmatic defaults to matching archives based on the + archive_name_format (see above). + example: sourcehostname + checks: + type: array + items: + type: object + required: ['name'] + additionalProperties: false + properties: + name: + type: string + enum: + - repository + - archives + - data + - extract + - disabled + description: | + Name of consistency check to run: "repository", + "archives", "data", and/or "extract". Set to "disabled" + to disable all consistency checks. "repository" checks + the consistency of the repository, "archives" checks all + of the archives, "data" verifies the integrity of the + data within the archives, and "extract" does an + extraction dry-run of the most recent archive. Note that + "data" implies "archives". + example: repository + frequency: + type: string + description: | + How frequently to run this type of consistency check (as + a best effort). The value is a number followed by a unit + of time. E.g., "2 weeks" to run this consistency check + no more than every two weeks for a given repository or + "1 month" to run it no more than monthly. Defaults to + "always": running this check every time checks are run. + example: 2 weeks + description: | + List of one or more consistency checks to run on a periodic basis + (if "frequency" is set) or every time borgmatic runs checks (if + "frequency" is omitted). + check_repositories: + type: array + items: + type: string + description: | + Paths or labels for a subset of the configured "repositories" (see + above) on which to run consistency checks. Handy in case some of + your repositories are very large, and so running consistency checks + on them would take too long. Defaults to running consistency checks + on all configured repositories. + example: + - user@backupserver:sourcehostname.borg + check_last: + type: integer + description: | + Restrict the number of checked archives to the last n. Applies only + to the "archives" check. Defaults to checking all archives. + example: 3 + color: + type: boolean + description: | + Apply color to console output. Can be overridden with --no-color + command-line flag. Defaults to true. + example: false + before_actions: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute before all + the actions for each repository. + example: + - "echo Starting actions." + before_backup: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute before + creating a backup, run once per repository. + example: + - "echo Starting a backup." + before_prune: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute before + pruning, run once per repository. + example: + - "echo Starting pruning." + before_compact: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute before + compaction, run once per repository. + example: + - "echo Starting compaction." + before_check: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute before + consistency checks, run once per repository. + example: + - "echo Starting checks." + before_extract: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute before + extracting a backup, run once per repository. + example: + - "echo Starting extracting." + after_backup: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute after + creating a backup, run once per repository. + example: + - "echo Finished a backup." + after_compact: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute after + compaction, run once per repository. + example: + - "echo Finished compaction." + after_prune: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute after + pruning, run once per repository. + example: + - "echo Finished pruning." + after_check: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute after + consistency checks, run once per repository. + example: + - "echo Finished checks." + after_extract: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute after + extracting a backup, run once per repository. + example: + - "echo Finished extracting." + after_actions: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute after all + actions for each repository. + example: + - "echo Finished actions." + on_error: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute when an + exception occurs during a "create", "prune", "compact", or "check" + action or an associated before/after hook. + example: + - "echo Error during create/prune/compact/check." + before_everything: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute before + running all actions (if one of them is "create"). These are + collected from all configuration files and then run once before all + of them (prior to all actions). + example: + - "echo Starting actions." + after_everything: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute after + running all actions (if one of them is "create"). These are + collected from all configuration files and then run once after all + of them (after any action). + example: + - "echo Completed actions." + postgresql_databases: + type: array + items: + type: object + required: ['name'] + additionalProperties: false + properties: + name: + type: string + description: | + Database name (required if using this hook). Or "all" to + dump all databases on the host. (Also set the "format" + to dump each database to a separate file instead of one + combined file.) Note that using this database hook + implicitly enables both read_special and one_file_system + (see above) to support dump and restore streaming. + example: users + hostname: + type: string + description: | + Database hostname to connect to. Defaults to connecting + via local Unix socket. + example: database.example.org + restore_hostname: + type: string + description: | + Database hostname to restore to. Defaults to the + "hostname" option. + example: database.example.org + port: + type: integer + description: Port to connect to. Defaults to 5432. + example: 5433 + restore_port: + type: integer + description: | + Port to restore to. Defaults to the "port" option. + example: 5433 + username: + type: string + description: | + Username with which to connect to the database. Defaults + to the username of the current user. You probably want + to specify the "postgres" superuser here when the + database name is "all". + example: dbuser + restore_username: + type: string + description: | + Username with which to restore the database. Defaults to + the "username" option. + example: dbuser + password: + type: string + description: | + Password with which to connect to the database. Omitting + a password will only work if PostgreSQL is configured to + trust the configured username without a password or you + create a ~/.pgpass file. + example: trustsome1 + restore_password: + type: string + description: | + Password with which to connect to the restore database. + Defaults to the "password" option. + example: trustsome1 + no_owner: + type: boolean + description: | + Do not output commands to set ownership of objects to + match the original database. By default, pg_dump and + pg_restore issue ALTER OWNER or SET SESSION + AUTHORIZATION statements to set ownership of created + schema elements. These statements will fail unless the + initial connection to the database is made by a + superuser. + example: true + format: + type: string + enum: ['plain', 'custom', 'directory', 'tar'] + description: | + Database dump output format. One of "plain", "custom", + "directory", or "tar". Defaults to "custom" (unlike raw + pg_dump) for a single database. Or, when database name + is "all" and format is blank, dumps all databases to a + single file. But if a format is specified with an "all" + database name, dumps each database to a separate file of + that format, allowing more convenient restores of + individual databases. See the pg_dump documentation for + more about formats. + example: directory + ssl_mode: + type: string + enum: ['disable', 'allow', 'prefer', + 'require', 'verify-ca', 'verify-full'] + description: | + SSL mode to use to connect to the database server. One + of "disable", "allow", "prefer", "require", "verify-ca" + or "verify-full". Defaults to "disable". + example: require + ssl_cert: + type: string + description: | + Path to a client certificate. + example: "/root/.postgresql/postgresql.crt" + ssl_key: + type: string + description: | + Path to a private client key. + example: "/root/.postgresql/postgresql.key" + ssl_root_cert: + type: string + description: | + Path to a root certificate containing a list of trusted + certificate authorities. + example: "/root/.postgresql/root.crt" + ssl_crl: + type: string + description: | + Path to a certificate revocation list. + example: "/root/.postgresql/root.crl" + pg_dump_command: + type: string + description: | + Command to use instead of "pg_dump" or "pg_dumpall". + This can be used to run a specific pg_dump version + (e.g., one inside a running container). Defaults to + "pg_dump" for single database dump or "pg_dumpall" to + dump all databases. + example: docker exec my_pg_container pg_dump + pg_restore_command: + type: string + description: | + Command to use instead of "pg_restore". This can be used + to run a specific pg_restore version (e.g., one inside a + running container). Defaults to "pg_restore". + example: docker exec my_pg_container pg_restore + psql_command: + type: string + description: | + Command to use instead of "psql". This can be used to + run a specific psql version (e.g., one inside a running + container). Defaults to "psql". + example: docker exec my_pg_container psql + options: + type: string + description: | + Additional pg_dump/pg_dumpall options to pass directly + to the dump command, without performing any validation + on them. See pg_dump documentation for details. + example: --role=someone + list_options: + type: string + description: | + Additional psql options to pass directly to the psql + command that lists available databases, without + performing any validation on them. See psql + documentation for details. + example: --role=someone + restore_options: + type: string + description: | + Additional pg_restore/psql options to pass directly to + the restore command, without performing any validation + on them. See pg_restore/psql documentation for details. + example: --role=someone + analyze_options: + type: string + description: | + Additional psql options to pass directly to the analyze + command run after a restore, without performing any + validation on them. See psql documentation for details. + example: --role=someone + description: | + List of one or more PostgreSQL databases to dump before creating a + backup, run once per configuration file. The database dumps are + added to your source directories at runtime, backed up, and removed + afterwards. Requires pg_dump/pg_dumpall/pg_restore commands. See + https://www.postgresql.org/docs/current/app-pgdump.html and + https://www.postgresql.org/docs/current/libpq-ssl.html for details. + mysql_databases: + type: array + items: + type: object + required: ['name'] + additionalProperties: false + properties: + name: + type: string + description: | + Database name (required if using this hook). Or "all" to + dump all databases on the host. Note that using this + database hook implicitly enables both read_special and + one_file_system (see above) to support dump and restore + streaming. + example: users + hostname: + type: string + description: | + Database hostname to connect to. Defaults to connecting + via local Unix socket. + example: database.example.org + restore_hostname: + type: string + description: | + Database hostname to restore to. Defaults to the + "hostname" option. + example: database.example.org + port: + type: integer + description: Port to connect to. Defaults to 3306. + example: 3307 + restore_port: + type: integer + description: | + Port to restore to. Defaults to the "port" option. + example: 5433 + username: + type: string + description: | + Username with which to connect to the database. Defaults + to the username of the current user. + example: dbuser + restore_username: + type: string + description: | + Username with which to restore the database. Defaults to + the "username" option. + example: dbuser + password: + type: string + description: | + Password with which to connect to the database. Omitting + a password will only work if MySQL is configured to + trust the configured username without a password. + example: trustsome1 + restore_password: + type: string + description: | + Password with which to connect to the restore database. + Defaults to the "password" option. + example: trustsome1 + format: + type: string + enum: ['sql'] + description: | + Database dump output format. Currently only "sql" is + supported. Defaults to "sql" for a single database. Or, + when database name is "all" and format is blank, dumps + all databases to a single file. But if a format is + specified with an "all" database name, dumps each + database to a separate file of that format, allowing + more convenient restores of individual databases. + example: directory + add_drop_database: + type: boolean + description: | + Use the "--add-drop-database" flag with mysqldump, + causing the database to be dropped right before restore. + Defaults to true. + example: false + options: + type: string + description: | + Additional mysqldump options to pass directly to the + dump command, without performing any validation on them. + See mysqldump documentation for details. + example: --skip-comments + list_options: + type: string + description: | + Additional mysql options to pass directly to the mysql + command that lists available databases, without + performing any validation on them. See mysql + documentation for details. + example: --defaults-extra-file=my.cnf + restore_options: + type: string + description: | + Additional mysql options to pass directly to the mysql + command that restores database dumps, without performing + any validation on them. See mysql documentation for + details. + example: --defaults-extra-file=my.cnf + description: | + List of one or more MySQL/MariaDB databases to dump before creating + a backup, run once per configuration file. The database dumps are + added to your source directories at runtime, backed up, and removed + afterwards. Requires mysqldump/mysql commands (from either MySQL or + MariaDB). See https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html + or https://mariadb.com/kb/en/library/mysqldump/ for details. + sqlite_databases: + type: array + items: + type: object + required: ['path','name'] + additionalProperties: false + properties: + name: + type: string + description: | + This is used to tag the database dump file with a name. + It is not the path to the database file itself. The name + "all" has no special meaning for SQLite databases. + example: users + path: + type: string + description: | + Path to the SQLite database file to dump. If relative, + it is relative to the current working directory. Note + that using this database hook implicitly enables both + read_special and one_file_system (see above) to support + dump and restore streaming. + example: /var/lib/sqlite/users.db + restore_path: + type: string + description: | + Path to the SQLite database file to restore to. Defaults + to the "path" option. + example: /var/lib/sqlite/users.db + mongodb_databases: + type: array + items: + type: object + required: ['name'] + additionalProperties: false + properties: + name: + type: string + description: | + Database name (required if using this hook). Or "all" to + dump all databases on the host. Note that using this + database hook implicitly enables both read_special and + one_file_system (see above) to support dump and restore + streaming. + example: users + hostname: + type: string + description: | + Database hostname to connect to. Defaults to connecting + to localhost. + example: database.example.org + restore_hostname: + type: string + description: | + Database hostname to restore to. Defaults to the + "hostname" option. + example: database.example.org + port: + type: integer + description: Port to connect to. Defaults to 27017. + example: 27018 + restore_port: + type: integer + description: | + Port to restore to. Defaults to the "port" option. + example: 5433 + username: + type: string + description: | + Username with which to connect to the database. Skip it + if no authentication is needed. + example: dbuser + restore_username: + type: string + description: | + Username with which to restore the database. Defaults to + the "username" option. + example: dbuser + password: + type: string + description: | + Password with which to connect to the database. Skip it + if no authentication is needed. + example: trustsome1 + restore_password: + type: string + description: | + Password with which to connect to the restore database. + Defaults to the "password" option. + example: trustsome1 + authentication_database: + type: string + description: | + Authentication database where the specified username + exists. If no authentication database is specified, the + database provided in "name" is used. If "name" is "all", + the "admin" database is used. + example: admin + format: + type: string + enum: ['archive', 'directory'] + description: | + Database dump output format. One of "archive", or + "directory". Defaults to "archive". See mongodump + documentation for details. Note that format is ignored + when the database name is "all". + example: directory + options: + type: string + description: | + Additional mongodump options to pass directly to the + dump command, without performing any validation on them. + See mongodump documentation for details. + example: --dumpDbUsersAndRoles + restore_options: + type: string + description: | + Additional mongorestore options to pass directly to the + dump command, without performing any validation on them. + See mongorestore documentation for details. + example: --restoreDbUsersAndRoles + description: | + List of one or more MongoDB databases to dump before creating a + backup, run once per configuration file. The database dumps are + added to your source directories at runtime, backed up, and removed + afterwards. Requires mongodump/mongorestore commands. See + https://docs.mongodb.com/database-tools/mongodump/ and + https://docs.mongodb.com/database-tools/mongorestore/ for details. + ntfy: + type: object + required: ['topic'] + additionalProperties: false + properties: + topic: + type: string + description: | + The topic to publish to. See https://ntfy.sh/docs/publish/ + for details. + example: topic + server: + type: string + description: | + The address of your self-hosted ntfy.sh instance. + example: https://ntfy.your-domain.com + username: + type: string + description: | + The username used for authentication. + example: testuser + password: + type: string + description: | + The password used for authentication. + example: fakepassword + start: + type: object + properties: + title: + type: string + description: | + The title of the message. + example: Ping! + message: + type: string + description: | + The message body to publish. + example: Your backups have failed. + priority: + type: string + description: | + The priority to set. + example: urgent + tags: + type: string + description: | + Tags to attach to the message. + example: incoming_envelope + finish: + type: object + properties: + title: + type: string + description: | + The title of the message. + example: Ping! + message: + type: string + description: | + The message body to publish. + example: Your backups have failed. + priority: + type: string + description: | + The priority to set. + example: urgent + tags: + type: string + description: | + Tags to attach to the message. + example: incoming_envelope + fail: + type: object + properties: + title: + type: string + description: | + The title of the message. + example: Ping! + message: + type: string + description: | + The message body to publish. + example: Your backups have failed. + priority: + type: string + description: | + The priority to set. + example: urgent + tags: + type: string + description: | + Tags to attach to the message. + example: incoming_envelope + states: type: array items: type: string + enum: + - start + - finish + - fail + uniqueItems: true description: | - List of source directories and files to backup. Globs and - tildes are expanded. Do not backslash spaces in path names. + List of one or more monitoring states to ping for: "start", + "finish", and/or "fail". Defaults to pinging for failure + only. example: - - /home - - /etc - - /var/log/syslog* - - /home/user/path with spaces - repositories: - type: array - items: - type: object - required: - - path - properties: - path: - type: string - example: ssh://user@backupserver/./{fqdn} - label: - type: string - example: backupserver - description: | - A required list of local or remote repositories with paths - and optional labels (which can be used with the --repository - flag to select a repository). Tildes are expanded. Multiple - repositories are backed up to in sequence. Borg placeholders - can be used. See the output of "borg help placeholders" for - details. See ssh_command for SSH options like identity file - or port. If systemd service is used, then add local - repository paths in the systemd service file to the - ReadWritePaths list. Prior to borgmatic 1.7.10, repositories - was a list of plain path strings. - example: - - path: ssh://user@backupserver/./sourcehostname.borg - label: backupserver - - path: /mnt/backup - label: local - working_directory: + - start + - finish + healthchecks: + type: object + required: ['ping_url'] + additionalProperties: false + properties: + ping_url: type: string description: | - Working directory for the "borg create" command. Tildes are - expanded. Useful for backing up using relative paths. See - http://borgbackup.readthedocs.io/en/stable/usage/create.html - for details. Defaults to not set. - example: /path/to/working/directory - one_file_system: + Healthchecks ping URL or UUID to notify when a backup + begins, ends, errors, or to send only logs. + example: https://hc-ping.com/your-uuid-here + verify_tls: type: boolean description: | - Stay in same file system: do not cross mount points beyond - the given source directories. Defaults to false. But when a - database hook is used, the setting here is ignored and - one_file_system is considered true. - example: true - numeric_ids: - type: boolean - description: | - Only store/extract numeric user and group identifiers. - Defaults to false. - example: true - atime: - type: boolean - description: | - Store atime into archive. Defaults to true in Borg < 1.2, - false in Borg 1.2+. - example: false - ctime: - type: boolean - description: Store ctime into archive. Defaults to true. - example: false - birthtime: - type: boolean - description: | - Store birthtime (creation date) into archive. Defaults to + Verify the TLS certificate of the ping URL host. Defaults to true. example: false - read_special: + send_logs: type: boolean description: | - Use Borg's --read-special flag to allow backup of block and - other special devices. Use with caution, as it will lead to - problems if used when backing up special devices such as - /dev/zero. Defaults to false. But when a database hook is - used, the setting here is ignored and read_special is - considered true. + Send borgmatic logs to Healthchecks as part the "finish", + "fail", and "log" states. Defaults to true. example: false - flags: - type: boolean + ping_body_limit: + type: integer description: | - Record filesystem flags (e.g. NODUMP, IMMUTABLE) in archive. - Defaults to true. - example: true - files_cache: - type: string - description: | - Mode in which to operate the files cache. See - http://borgbackup.readthedocs.io/en/stable/usage/create.html - for details. Defaults to "ctime,size,inode". - example: ctime,size,inode - local_path: - type: string - description: | - Alternate Borg local executable. Defaults to "borg". - example: borg1 - remote_path: - type: string - description: | - Alternate Borg remote executable. Defaults to "borg". - example: borg1 - patterns: + Number of bytes of borgmatic logs to send to Healthchecks, + ideally the same as PING_BODY_LIMIT configured on the + Healthchecks server. Set to 0 to send all logs and disable + this truncation. Defaults to 100000. + example: 200000 + states: type: array items: type: string + enum: + - start + - finish + - fail + - log + uniqueItems: true description: | - Any paths matching these patterns are included/excluded from - backups. Globs are expanded. (Tildes are not.) See the - output of "borg help patterns" for more details. Quote any - value if it contains leading punctuation, so it parses - correctly. Note that only one of "patterns" and - "source_directories" may be used. + List of one or more monitoring states to ping for: "start", + "finish", "fail", and/or "log". Defaults to pinging for all + states. example: - - 'R /' - - '- /home/*/.cache' - - '+ /home/susan' - - '- /home/*' - patterns_from: - type: array - items: - type: string - description: | - Read include/exclude patterns from one or more separate - named files, one pattern per line. Note that Borg considers - this option experimental. See the output of "borg help - patterns" for more details. - example: - - /etc/borgmatic/patterns - exclude_patterns: - type: array - items: - type: string - description: | - Any paths matching these patterns are excluded from backups. - Globs and tildes are expanded. Note that a glob pattern must - either start with a glob or be an absolute path. Do not - backslash spaces in path names. See the output of "borg help - patterns" for more details. - example: - - '*.pyc' - - /home/*/.cache - - '*/.vim*.tmp' - - /etc/ssl - - /home/user/path with spaces - exclude_from: - type: array - items: - type: string - description: | - Read exclude patterns from one or more separate named files, - one pattern per line. See the output of "borg help patterns" - for more details. - example: - - /etc/borgmatic/excludes - exclude_caches: - type: boolean - description: | - Exclude directories that contain a CACHEDIR.TAG file. See - http://www.brynosaurus.com/cachedir/spec.html for details. - Defaults to false. - example: true - exclude_if_present: - type: array - items: - type: string - description: | - Exclude directories that contain a file with the given - filenames. Defaults to not set. - example: - - .nobackup - keep_exclude_tags: - type: boolean - description: | - If true, the exclude_if_present filename is included in - backups. Defaults to false, meaning that the - exclude_if_present filename is omitted from backups. - example: true - exclude_nodump: - type: boolean - description: | - Exclude files with the NODUMP flag. Defaults to false. - example: true - borgmatic_source_directory: - type: string - description: | - Path for additional source files used for temporary internal - state like borgmatic database dumps. Note that changing this - path prevents "borgmatic restore" from finding any database - dumps created before the change. Defaults to ~/.borgmatic - example: /tmp/borgmatic - source_directories_must_exist: - type: boolean - description: | - If true, then source directories must exist, otherwise an - error is raised. Defaults to false. - example: true - storage: - type: object + - finish description: | - Repository storage options. See - https://borgbackup.readthedocs.io/en/stable/usage/create.html and - https://borgbackup.readthedocs.io/en/stable/usage/general.html for - details. + Configuration for a monitoring integration with Healthchecks. Create + an account at https://healthchecks.io (or self-host Healthchecks) if + you'd like to use this service. See borgmatic monitoring + documentation for details. + cronitor: + type: object + required: ['ping_url'] additionalProperties: false properties: - encryption_passcommand: + ping_url: type: string description: | - The standard output of this command is used to unlock the - encryption key. Only use on repositories that were - initialized with passcommand/repokey/keyfile encryption. - Note that if both encryption_passcommand and - encryption_passphrase are set, then encryption_passphrase - takes precedence. Defaults to not set. - example: "secret-tool lookup borg-repository repo-name" - encryption_passphrase: - type: string - description: | - Passphrase to unlock the encryption key with. Only use on - repositories that were initialized with - passphrase/repokey/keyfile encryption. Quote the value if it - contains punctuation, so it parses correctly. And backslash - any quote or backslash literals as well. Defaults to not - set. - example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" - checkpoint_interval: - type: integer - description: | - Number of seconds between each checkpoint during a - long-running backup. See - https://borgbackup.readthedocs.io/en/stable/faq.html - for details. Defaults to checkpoints every 1800 seconds (30 - minutes). - example: 1800 - checkpoint_volume: - type: integer - description: | - Number of backed up bytes between each checkpoint during a - long-running backup. Only supported with Borg 2+. See - https://borgbackup.readthedocs.io/en/stable/faq.html - for details. Defaults to only time-based checkpointing (see - "checkpoint_interval") instead of volume-based - checkpointing. - example: 1048576 - chunker_params: - type: string - description: | - Specify the parameters passed to then chunker - (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, - HASH_WINDOW_SIZE). See - https://borgbackup.readthedocs.io/en/stable/internals.html - for details. Defaults to "19,23,21,4095". - example: 19,23,21,4095 - compression: - type: string - description: | - Type of compression to use when creating archives. See - http://borgbackup.readthedocs.io/en/stable/usage/create.html - for details. Defaults to "lz4". - example: lz4 - upload_rate_limit: - type: integer - description: | - Remote network upload rate limit in kiBytes/second. Defaults - to unlimited. - example: 100 - retries: - type: integer - description: | - Number of times to retry a failing backup before giving up. - Defaults to 0 (i.e., does not attempt retry). - example: 3 - retry_wait: - type: integer - description: | - Wait time between retries (in seconds) to allow transient - issues to pass. Increases after each retry as a form of - backoff. Defaults to 0 (no wait). - example: 10 - temporary_directory: - type: string - description: | - Directory where temporary files are stored. Defaults to - $TMPDIR - example: /path/to/tmpdir - ssh_command: - type: string - description: | - Command to use instead of "ssh". This can be used to specify - ssh options. Defaults to not set. - example: ssh -i /path/to/private/key - borg_base_directory: - type: string - description: | - Base path used for various Borg directories. Defaults to - $HOME, ~$USER, or ~. - example: /path/to/base - borg_config_directory: - type: string - description: | - Path for Borg configuration files. Defaults to - $borg_base_directory/.config/borg - example: /path/to/base/config - borg_cache_directory: - type: string - description: | - Path for Borg cache files. Defaults to - $borg_base_directory/.cache/borg - example: /path/to/base/cache - borg_files_cache_ttl: - type: integer - description: | - Maximum time to live (ttl) for entries in the Borg files - cache. - example: 20 - borg_security_directory: - type: string - description: | - Path for Borg security and encryption nonce files. Defaults - to $borg_base_directory/.config/borg/security - example: /path/to/base/config/security - borg_keys_directory: - type: string - description: | - Path for Borg encryption key files. Defaults to - $borg_base_directory/.config/borg/keys - example: /path/to/base/config/keys - umask: - type: integer - description: Umask to be used for borg create. Defaults to 0077. - example: 0077 - lock_wait: - type: integer - description: | - Maximum seconds to wait for acquiring a repository/cache - lock. Defaults to 1. - example: 5 - archive_name_format: - type: string - description: | - Name of the archive. Borg placeholders can be used. See the - output of "borg help placeholders" for details. Defaults to - "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". When running - actions like rlist, info, or check, borgmatic automatically - tries to match only archives created with this name format. - example: "{hostname}-documents-{now}" - match_archives: - type: string - description: | - A Borg pattern for filtering down the archives used by - borgmatic actions that operate on multiple archives. For - Borg 1.x, use a shell pattern here and see the output of - "borg help placeholders" for details. For Borg 2.x, see the - output of "borg help match-archives". If match_archives is - not specified, borgmatic defaults to deriving the - match_archives value from archive_name_format. - example: "sh:{hostname}-*" - relocated_repo_access_is_ok: - type: boolean - description: | - Bypass Borg error about a repository that has been moved. - Defaults to false. - example: true - unknown_unencrypted_repo_access_is_ok: - type: boolean - description: | - Bypass Borg error about a previously unknown unencrypted - repository. Defaults to false. - example: true - extra_borg_options: - type: object - additionalProperties: false - properties: - init: - type: string - description: | - Extra command-line options to pass to "borg init". - example: "--extra-option" - create: - type: string - description: | - Extra command-line options to pass to "borg create". - example: "--extra-option" - prune: - type: string - description: | - Extra command-line options to pass to "borg prune". - example: "--extra-option" - compact: - type: string - description: | - Extra command-line options to pass to "borg compact". - example: "--extra-option" - check: - type: string - description: | - Extra command-line options to pass to "borg check". - example: "--extra-option" - description: | - Additional options to pass directly to particular Borg - commands, handy for Borg options that borgmatic does not yet - support natively. Note that borgmatic does not perform any - validation on these options. Running borgmatic with - "--verbosity 2" shows the exact Borg command-line - invocation. - retention: - type: object + Cronitor ping URL to notify when a backup begins, + ends, or errors. + example: https://cronitor.link/d3x0c1 description: | - Retention policy for how many backups to keep in each category. See - https://borgbackup.readthedocs.io/en/stable/usage/prune.html for - details. At least one of the "keep" options is required for pruning - to work. To skip pruning entirely, run "borgmatic create" or "check" - without the "prune" action. See borgmatic documentation for details. + Configuration for a monitoring integration with Cronitor. Create an + account at https://cronitor.io if you'd like to use this service. + See borgmatic monitoring documentation for details. + pagerduty: + type: object + required: ['integration_key'] additionalProperties: false properties: - keep_within: - type: string - description: Keep all archives within this time interval. - example: 3H - keep_secondly: - type: integer - description: Number of secondly archives to keep. - example: 60 - keep_minutely: - type: integer - description: Number of minutely archives to keep. - example: 60 - keep_hourly: - type: integer - description: Number of hourly archives to keep. - example: 24 - keep_daily: - type: integer - description: Number of daily archives to keep. - example: 7 - keep_weekly: - type: integer - description: Number of weekly archives to keep. - example: 4 - keep_monthly: - type: integer - description: Number of monthly archives to keep. - example: 6 - keep_yearly: - type: integer - description: Number of yearly archives to keep. - example: 1 - prefix: + integration_key: type: string description: | - Deprecated. When pruning, only consider archive names - starting with this prefix. Borg placeholders can be used. - See the output of "borg help placeholders" for details. - If a prefix is not specified, borgmatic defaults to - matching archives based on the archive_name_format (see - above). - example: sourcehostname - consistency: - type: object + PagerDuty integration key used to notify PagerDuty + when a backup errors. + example: a177cad45bd374409f78906a810a3074 description: | - Consistency checks to run after backups. See - https://borgbackup.readthedocs.io/en/stable/usage/check.html and - https://borgbackup.readthedocs.io/en/stable/usage/extract.html for - details. + Configuration for a monitoring integration with PagerDuty. Create an + account at https://www.pagerduty.com/ if you'd like to use this + service. See borgmatic monitoring documentation for details. + cronhub: + type: object + required: ['ping_url'] additionalProperties: false properties: - checks: - type: array - items: - type: object - required: ['name'] - additionalProperties: false - properties: - name: - type: string - enum: - - repository - - archives - - data - - extract - - disabled - description: | - Name of consistency check to run: "repository", - "archives", "data", and/or "extract". Set to - "disabled" to disable all consistency checks. - "repository" checks the consistency of the - repository, "archives" checks all of the - archives, "data" verifies the integrity of the - data within the archives, and "extract" does an - extraction dry-run of the most recent archive. - Note that "data" implies "archives". - example: repository - frequency: - type: string - description: | - How frequently to run this type of consistency - check (as a best effort). The value is a number - followed by a unit of time. E.g., "2 weeks" to - run this consistency check no more than every - two weeks for a given repository or "1 month" to - run it no more than monthly. Defaults to - "always": running this check every time checks - are run. - example: 2 weeks - description: | - List of one or more consistency checks to run on a periodic - basis (if "frequency" is set) or every time borgmatic runs - checks (if "frequency" is omitted). - check_repositories: - type: array - items: - type: string - description: | - Paths or labels for a subset of the repositories in the - location section on which to run consistency checks. Handy - in case some of your repositories are very large, and so - running consistency checks on them would take too long. - Defaults to running consistency checks on all repositories - configured in the location section. - example: - - user@backupserver:sourcehostname.borg - check_last: - type: integer - description: | - Restrict the number of checked archives to the last n. - Applies only to the "archives" check. Defaults to checking - all archives. - example: 3 - prefix: + ping_url: type: string description: | - Deprecated. When performing the "archives" check, only - consider archive names starting with this prefix. Borg - placeholders can be used. See the output of "borg help - placeholders" for details. If a prefix is not specified, - borgmatic defaults to matching archives based on the - archive_name_format (see above). - example: sourcehostname - output: - type: object + Cronhub ping URL to notify when a backup begins, + ends, or errors. + example: https://cronhub.io/ping/1f5e3410-254c-5587 description: | - Options for customizing borgmatic's own output and logging. - additionalProperties: false - properties: - color: - type: boolean - description: | - Apply color to console output. Can be overridden with - --no-color command-line flag. Defaults to true. - example: false - hooks: - type: object - description: | - Shell commands, scripts, or integrations to execute at various - points during a borgmatic run. IMPORTANT: All provided commands and - scripts are executed with user permissions of borgmatic. Do not - forget to set secure permissions on this configuration file (chmod - 0600) as well as on any script called from a hook (chmod 0700) to - prevent potential shell injection or privilege escalation. - additionalProperties: false - properties: - before_actions: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - before all the actions for each repository. - example: - - "echo Starting actions." - before_backup: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - before creating a backup, run once per repository. - example: - - "echo Starting a backup." - before_prune: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - before pruning, run once per repository. - example: - - "echo Starting pruning." - before_compact: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - before compaction, run once per repository. - example: - - "echo Starting compaction." - before_check: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - before consistency checks, run once per repository. - example: - - "echo Starting checks." - before_extract: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - before extracting a backup, run once per repository. - example: - - "echo Starting extracting." - after_backup: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - after creating a backup, run once per repository. - example: - - "echo Finished a backup." - after_compact: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - after compaction, run once per repository. - example: - - "echo Finished compaction." - after_prune: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - after pruning, run once per repository. - example: - - "echo Finished pruning." - after_check: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - after consistency checks, run once per repository. - example: - - "echo Finished checks." - after_extract: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - after extracting a backup, run once per repository. - example: - - "echo Finished extracting." - after_actions: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - after all actions for each repository. - example: - - "echo Finished actions." - on_error: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - when an exception occurs during a "create", "prune", - "compact", or "check" action or an associated before/after - hook. - example: - - "echo Error during create/prune/compact/check." - before_everything: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - before running all actions (if one of them is "create"). - These are collected from all configuration files and then - run once before all of them (prior to all actions). - example: - - "echo Starting actions." - after_everything: - type: array - items: - type: string - description: | - List of one or more shell commands or scripts to execute - after running all actions (if one of them is "create"). - These are collected from all configuration files and then - run once after all of them (after any action). - example: - - "echo Completed actions." - postgresql_databases: - type: array - items: - type: object - required: ['name'] - additionalProperties: false - properties: - name: - type: string - description: | - Database name (required if using this hook). Or - "all" to dump all databases on the host. (Also - set the "format" to dump each database to a - separate file instead of one combined file.) - Note that using this database hook implicitly - enables both read_special and one_file_system - (see above) to support dump and restore - streaming. - example: users - hostname: - type: string - description: | - Database hostname to connect to. Defaults to - connecting via local Unix socket. - example: database.example.org - restore_hostname: - type: string - description: | - Database hostname to restore to. Defaults to - the "hostname" option. - example: database.example.org - port: - type: integer - description: Port to connect to. Defaults to 5432. - example: 5433 - restore_port: - type: integer - description: Port to restore to. Defaults to the - "port" option. - example: 5433 - username: - type: string - description: | - Username with which to connect to the database. - Defaults to the username of the current user. - You probably want to specify the "postgres" - superuser here when the database name is "all". - example: dbuser - restore_username: - type: string - description: | - Username with which to restore the database. - Defaults to the "username" option. - example: dbuser - password: - type: string - description: | - Password with which to connect to the database. - Omitting a password will only work if PostgreSQL - is configured to trust the configured username - without a password or you create a ~/.pgpass - file. - example: trustsome1 - restore_password: - type: string - description: | - Password with which to connect to the restore - database. Defaults to the "password" option. - example: trustsome1 - no_owner: - type: boolean - description: | - Do not output commands to set ownership of - objects to match the original database. By - default, pg_dump and pg_restore issue ALTER - OWNER or SET SESSION AUTHORIZATION statements - to set ownership of created schema elements. - These statements will fail unless the initial - connection to the database is made by a - superuser. - example: true - format: - type: string - enum: ['plain', 'custom', 'directory', 'tar'] - description: | - Database dump output format. One of "plain", - "custom", "directory", or "tar". Defaults to - "custom" (unlike raw pg_dump) for a single - database. Or, when database name is "all" and - format is blank, dumps all databases to a single - file. But if a format is specified with an "all" - database name, dumps each database to a separate - file of that format, allowing more convenient - restores of individual databases. See the - pg_dump documentation for more about formats. - example: directory - ssl_mode: - type: string - enum: ['disable', 'allow', 'prefer', - 'require', 'verify-ca', 'verify-full'] - description: | - SSL mode to use to connect to the database - server. One of "disable", "allow", "prefer", - "require", "verify-ca" or "verify-full". - Defaults to "disable". - example: require - ssl_cert: - type: string - description: | - Path to a client certificate. - example: "/root/.postgresql/postgresql.crt" - ssl_key: - type: string - description: | - Path to a private client key. - example: "/root/.postgresql/postgresql.key" - ssl_root_cert: - type: string - description: | - Path to a root certificate containing a list of - trusted certificate authorities. - example: "/root/.postgresql/root.crt" - ssl_crl: - type: string - description: | - Path to a certificate revocation list. - example: "/root/.postgresql/root.crl" - pg_dump_command: - type: string - description: | - Command to use instead of "pg_dump" or - "pg_dumpall". This can be used to run a specific - pg_dump version (e.g., one inside a running - container). Defaults to "pg_dump" for single - database dump or "pg_dumpall" to dump all - databases. - example: docker exec my_pg_container pg_dump - pg_restore_command: - type: string - description: | - Command to use instead of "pg_restore". This - can be used to run a specific pg_restore - version (e.g., one inside a running container). - Defaults to "pg_restore". - example: docker exec my_pg_container pg_restore - psql_command: - type: string - description: | - Command to use instead of "psql". This can be - used to run a specific psql version (e.g., - one inside a running container). Defaults to - "psql". - example: docker exec my_pg_container psql - options: - type: string - description: | - Additional pg_dump/pg_dumpall options to pass - directly to the dump command, without performing - any validation on them. See pg_dump - documentation for details. - example: --role=someone - list_options: - type: string - description: | - Additional psql options to pass directly to the - psql command that lists available databases, - without performing any validation on them. See - psql documentation for details. - example: --role=someone - restore_options: - type: string - description: | - Additional pg_restore/psql options to pass - directly to the restore command, without - performing any validation on them. See - pg_restore/psql documentation for details. - example: --role=someone - analyze_options: - type: string - description: | - Additional psql options to pass directly to the - analyze command run after a restore, without - performing any validation on them. See psql - documentation for details. - example: --role=someone - description: | - List of one or more PostgreSQL databases to dump before - creating a backup, run once per configuration file. The - database dumps are added to your source directories at - runtime, backed up, and removed afterwards. Requires - pg_dump/pg_dumpall/pg_restore commands. See - https://www.postgresql.org/docs/current/app-pgdump.html and - https://www.postgresql.org/docs/current/libpq-ssl.html for - details. - mysql_databases: - type: array - items: - type: object - required: ['name'] - additionalProperties: false - properties: - name: - type: string - description: | - Database name (required if using this hook). Or - "all" to dump all databases on the host. Note - that using this database hook implicitly enables - both read_special and one_file_system (see - above) to support dump and restore streaming. - example: users - hostname: - type: string - description: | - Database hostname to connect to. Defaults to - connecting via local Unix socket. - example: database.example.org - restore_hostname: - type: string - description: | - Database hostname to restore to. Defaults to - the "hostname" option. - example: database.example.org - port: - type: integer - description: Port to connect to. Defaults to 3306. - example: 3307 - restore_port: - type: integer - description: Port to restore to. Defaults to the - "port" option. - example: 5433 - username: - type: string - description: | - Username with which to connect to the database. - Defaults to the username of the current user. - example: dbuser - restore_username: - type: string - description: | - Username with which to restore the database. - Defaults to the "username" option. - example: dbuser - password: - type: string - description: | - Password with which to connect to the database. - Omitting a password will only work if MySQL is - configured to trust the configured username - without a password. - example: trustsome1 - restore_password: - type: string - description: | - Password with which to connect to the restore - database. Defaults to the "password" option. - example: trustsome1 - format: - type: string - enum: ['sql'] - description: | - Database dump output format. Currently only - "sql" is supported. Defaults to "sql" for a - single database. Or, when database name is "all" - and format is blank, dumps all databases to a - single file. But if a format is specified with - an "all" database name, dumps each database to a - separate file of that format, allowing more - convenient restores of individual databases. - example: directory - add_drop_database: - type: boolean - description: | - Use the "--add-drop-database" flag with - mysqldump, causing the database to be dropped - right before restore. Defaults to true. - example: false - options: - type: string - description: | - Additional mysqldump options to pass directly to - the dump command, without performing any - validation on them. See mysqldump documentation - for details. - example: --skip-comments - list_options: - type: string - description: | - Additional mysql options to pass directly to - the mysql command that lists available - databases, without performing any validation on - them. See mysql documentation for details. - example: --defaults-extra-file=my.cnf - restore_options: - type: string - description: | - Additional mysql options to pass directly to - the mysql command that restores database dumps, - without performing any validation on them. See - mysql documentation for details. - example: --defaults-extra-file=my.cnf - description: | - List of one or more MySQL/MariaDB databases to dump before - creating a backup, run once per configuration file. The - database dumps are added to your source directories at - runtime, backed up, and removed afterwards. Requires - mysqldump/mysql commands (from either MySQL or MariaDB). See - https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or - https://mariadb.com/kb/en/library/mysqldump/ for details. - sqlite_databases: - type: array - items: - type: object - required: ['path','name'] - additionalProperties: false - properties: - name: - type: string - description: | - This is used to tag the database dump file - with a name. It is not the path to the database - file itself. The name "all" has no special - meaning for SQLite databases. - example: users - path: - type: string - description: | - Path to the SQLite database file to dump. If - relative, it is relative to the current working - directory. Note that using this - database hook implicitly enables both - read_special and one_file_system (see above) to - support dump and restore streaming. - example: /var/lib/sqlite/users.db - restore_path: - type: string - description: | - Path to the SQLite database file to restore to. - Defaults to the "path" option. - example: /var/lib/sqlite/users.db - mongodb_databases: - type: array - items: - type: object - required: ['name'] - additionalProperties: false - properties: - name: - type: string - description: | - Database name (required if using this hook). Or - "all" to dump all databases on the host. Note - that using this database hook implicitly enables - both read_special and one_file_system (see - above) to support dump and restore streaming. - example: users - hostname: - type: string - description: | - Database hostname to connect to. Defaults to - connecting to localhost. - example: database.example.org - restore_hostname: - type: string - description: | - Database hostname to restore to. Defaults to - the "hostname" option. - example: database.example.org - port: - type: integer - description: Port to connect to. Defaults to 27017. - example: 27018 - restore_port: - type: integer - description: Port to restore to. Defaults to the - "port" option. - example: 5433 - username: - type: string - description: | - Username with which to connect to the database. - Skip it if no authentication is needed. - example: dbuser - restore_username: - type: string - description: | - Username with which to restore the database. - Defaults to the "username" option. - example: dbuser - password: - type: string - description: | - Password with which to connect to the database. - Skip it if no authentication is needed. - example: trustsome1 - restore_password: - type: string - description: | - Password with which to connect to the restore - database. Defaults to the "password" option. - example: trustsome1 - authentication_database: - type: string - description: | - Authentication database where the specified - username exists. If no authentication database - is specified, the database provided in "name" - is used. If "name" is "all", the "admin" - database is used. - example: admin - format: - type: string - enum: ['archive', 'directory'] - description: | - Database dump output format. One of "archive", - or "directory". Defaults to "archive". See - mongodump documentation for details. Note that - format is ignored when the database name is - "all". - example: directory - options: - type: string - description: | - Additional mongodump options to pass - directly to the dump command, without performing - any validation on them. See mongodump - documentation for details. - example: --dumpDbUsersAndRoles - restore_options: - type: string - description: | - Additional mongorestore options to pass - directly to the dump command, without performing - any validation on them. See mongorestore - documentation for details. - example: --restoreDbUsersAndRoles - description: | - List of one or more MongoDB databases to dump before - creating a backup, run once per configuration file. The - database dumps are added to your source directories at - runtime, backed up, and removed afterwards. Requires - mongodump/mongorestore commands. See - https://docs.mongodb.com/database-tools/mongodump/ and - https://docs.mongodb.com/database-tools/mongorestore/ for - details. - ntfy: - type: object - required: ['topic'] - additionalProperties: false - properties: - topic: - type: string - description: | - The topic to publish to. - (https://ntfy.sh/docs/publish/) - example: topic - server: - type: string - description: | - The address of your self-hosted ntfy.sh instance. - example: https://ntfy.your-domain.com - username: - type: string - description: | - The username used for authentication. - example: testuser - password: - type: string - description: | - The password used for authentication. - example: fakepassword - start: - type: object - properties: - title: - type: string - description: | - The title of the message - example: Ping! - message: - type: string - description: | - The message body to publish. - example: Your backups have failed. - priority: - type: string - description: | - The priority to set. - example: urgent - tags: - type: string - description: | - Tags to attach to the message. - example: incoming_envelope - finish: - type: object - properties: - title: - type: string - description: | - The title of the message. - example: Ping! - message: - type: string - description: | - The message body to publish. - example: Your backups have failed. - priority: - type: string - description: | - The priority to set. - example: urgent - tags: - type: string - description: | - Tags to attach to the message. - example: incoming_envelope - fail: - type: object - properties: - title: - type: string - description: | - The title of the message. - example: Ping! - message: - type: string - description: | - The message body to publish. - example: Your backups have failed. - priority: - type: string - description: | - The priority to set. - example: urgent - tags: - type: string - description: | - Tags to attach to the message. - example: incoming_envelope - states: - type: array - items: - type: string - enum: - - start - - finish - - fail - uniqueItems: true - description: | - List of one or more monitoring states to ping for: - "start", "finish", and/or "fail". Defaults to - pinging for failure only. - example: - - start - - finish - healthchecks: - type: object - required: ['ping_url'] - additionalProperties: false - properties: - ping_url: - type: string - description: | - Healthchecks ping URL or UUID to notify when a - backup begins, ends, errors, or to send only logs. - example: https://hc-ping.com/your-uuid-here - verify_tls: - type: boolean - description: | - Verify the TLS certificate of the ping URL host. - Defaults to true. - example: false - send_logs: - type: boolean - description: | - Send borgmatic logs to Healthchecks as part the - "finish", "fail", and "log" states. Defaults to - true. - example: false - ping_body_limit: - type: integer - description: | - Number of bytes of borgmatic logs to send to - Healthchecks, ideally the same as PING_BODY_LIMIT - configured on the Healthchecks server. Set to 0 to - send all logs and disable this truncation. Defaults - to 100000. - example: 200000 - states: - type: array - items: - type: string - enum: - - start - - finish - - fail - - log - uniqueItems: true - description: | - List of one or more monitoring states to ping for: - "start", "finish", "fail", and/or "log". Defaults to - pinging for all states. - example: - - finish - description: | - Configuration for a monitoring integration with - Healthchecks. Create an account at https://healthchecks.io - (or self-host Healthchecks) if you'd like to use this - service. See borgmatic monitoring documentation for details. - cronitor: - type: object - required: ['ping_url'] - additionalProperties: false - properties: - ping_url: - type: string - description: | - Cronitor ping URL to notify when a backup begins, - ends, or errors. - example: https://cronitor.link/d3x0c1 - description: | - Configuration for a monitoring integration with Cronitor. - Create an account at https://cronitor.io if you'd - like to use this service. See borgmatic monitoring - documentation for details. - pagerduty: - type: object - required: ['integration_key'] - additionalProperties: false - properties: - integration_key: - type: string - description: | - PagerDuty integration key used to notify PagerDuty - when a backup errors. - example: a177cad45bd374409f78906a810a3074 - description: | - Configuration for a monitoring integration with PagerDuty. - Create an account at https://www.pagerduty.com/ if you'd - like to use this service. See borgmatic monitoring - documentation for details. - cronhub: - type: object - required: ['ping_url'] - additionalProperties: false - properties: - ping_url: - type: string - description: | - Cronhub ping URL to notify when a backup begins, - ends, or errors. - example: https://cronhub.io/ping/1f5e3410-254c-5587 - description: | - Configuration for a monitoring integration with Crunhub. - Create an account at https://cronhub.io if you'd like to - use this service. See borgmatic monitoring documentation - for details. - umask: - type: integer - description: | - Umask used when executing hooks. Defaults to the umask that - borgmatic is run with. - example: 0077 + Configuration for a monitoring integration with Crunhub. Create an + account at https://cronhub.io if you'd like to use this service. See + borgmatic monitoring documentation for details. diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 5835ead1..c13329f3 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -71,18 +71,15 @@ def apply_logical_validation(config_filename, parsed_configuration): below), run through any additional logical validation checks. If there are any such validation problems, raise a Validation_error. ''' - location_repositories = parsed_configuration.get('location', {}).get('repositories') - check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) + repositories = parsed_configuration.get('repositories') + check_repositories = parsed_configuration.get('check_repositories', []) for repository in check_repositories: if not any( - repositories_match(repository, config_repository) - for config_repository in location_repositories + repositories_match(repository, config_repository) for config_repository in repositories ): raise Validation_error( config_filename, - ( - f'Unknown repository in the "consistency" section\'s "check_repositories": {repository}', - ), + (f'Unknown repository in "check_repositories": {repository}',), ) @@ -90,11 +87,15 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML rendition of JSON Schema format, a sequence of configuration file override strings in the form - of "section.option=value", return the parsed configuration as a data structure of nested dicts + of "option.suboption=value", return the parsed configuration as a data structure of nested dicts and lists corresponding to the schema. Example return value: - {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, - 'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}} + { + 'source_directories': ['/home', '/etc'], + 'repository': 'hostname.borg', + 'keep_daily': 7, + 'checks': ['repository', 'archives'], + } Also return a sequence of logging.LogRecord instances containing any warnings about the configuration. @@ -174,7 +175,7 @@ def guard_configuration_contains_repository(repository, configurations): tuple( config_repository for config in configurations.values() - for config_repository in config['location']['repositories'] + for config_repository in config['repositories'] if repositories_match(config_repository, repository) ) ) @@ -198,7 +199,7 @@ def guard_single_repository_selected(repository, configurations): tuple( config_repository for config in configurations.values() - for config_repository in config['location']['repositories'] + for config_repository in config['repositories'] ) ) diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index 05ada575..170f1916 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -22,7 +22,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): ''' Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index d669c09d..d57920cd 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -22,7 +22,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): ''' Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index fa7bd9b3..d98473ab 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -27,18 +27,17 @@ HOOK_NAME_TO_MODULE = { } -def call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs): +def call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs): ''' - Given the hooks configuration dict and a prefix to use in log entries, call the requested - function of the Python module corresponding to the given hook name. Supply that call with the - configuration for this hook (if any), the log prefix, and any given args and kwargs. Return any - return value. + Given a configuration dict and a prefix to use in log entries, call the requested function of + the Python module corresponding to the given hook name. Supply that call with the configuration + for this hook (if any), the log prefix, and any given args and kwargs. Return any return value. Raise ValueError if the hook name is unknown. Raise AttributeError if the function name is not found in the module. Raise anything else that the called function raises. ''' - config = hooks.get(hook_name, {}) + hook_config = config.get(hook_name, {}) try: module = HOOK_NAME_TO_MODULE[hook_name] @@ -46,15 +45,15 @@ def call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs): raise ValueError(f'Unknown hook name: {hook_name}') logger.debug(f'{log_prefix}: Calling {hook_name} hook function {function_name}') - return getattr(module, function_name)(config, log_prefix, *args, **kwargs) + return getattr(module, function_name)(hook_config, config, log_prefix, *args, **kwargs) -def call_hooks(function_name, hooks, log_prefix, hook_names, *args, **kwargs): +def call_hooks(function_name, config, log_prefix, hook_names, *args, **kwargs): ''' - Given the hooks configuration dict and a prefix to use in log entries, call the requested - function of the Python module corresponding to each given hook name. Supply each call with the - configuration for that hook, the log prefix, and any given args and kwargs. Collect any return - values into a dict from hook name to return value. + Given a configuration dict and a prefix to use in log entries, call the requested function of + the Python module corresponding to each given hook name. Supply each call with the configuration + for that hook, the log prefix, and any given args and kwargs. Collect any return values into a + dict from hook name to return value. If the hook name is not present in the hooks configuration, then don't call the function for it and omit it from the return values. @@ -64,23 +63,23 @@ def call_hooks(function_name, hooks, log_prefix, hook_names, *args, **kwargs): Raise anything else that a called function raises. An error stops calls to subsequent functions. ''' return { - hook_name: call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs) + hook_name: call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs) for hook_name in hook_names - if hooks.get(hook_name) + if config.get(hook_name) } -def call_hooks_even_if_unconfigured(function_name, hooks, log_prefix, hook_names, *args, **kwargs): +def call_hooks_even_if_unconfigured(function_name, config, log_prefix, hook_names, *args, **kwargs): ''' - Given the hooks configuration dict and a prefix to use in log entries, call the requested - function of the Python module corresponding to each given hook name. Supply each call with the - configuration for that hook, the log prefix, and any given args and kwargs. Collect any return - values into a dict from hook name to return value. + Given a configuration dict and a prefix to use in log entries, call the requested function of + the Python module corresponding to each given hook name. Supply each call with the configuration + for that hook, the log prefix, and any given args and kwargs. Collect any return values into a + dict from hook name to return value. Raise AttributeError if the function name is not found in the module. Raise anything else that a called function raises. An error stops calls to subsequent functions. ''' return { - hook_name: call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs) + hook_name: call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs) for hook_name in hook_names } diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index 4cafc49f..511e9566 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -90,7 +90,7 @@ def initialize_monitor(hook_config, config_filename, monitoring_log_level, dry_r ) -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): ''' Ping the configured Healthchecks URL or UUID, modified with the monitor.State. Use the given configuration filename in any log entries, and log to Healthchecks with the giving log level. diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index f8899268..3c91a183 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -6,21 +6,20 @@ from borgmatic.hooks import dump logger = logging.getLogger(__name__) -def make_dump_path(location_config): # pragma: no cover +def make_dump_path(config): # pragma: no cover ''' - Make the dump path from the given location configuration and the name of this hook. + Make the dump path from the given configuration dict and the name of this hook. ''' return dump.make_database_dump_path( - location_config.get('borgmatic_source_directory'), 'mongodb_databases' + config.get('borgmatic_source_directory'), 'mongodb_databases' ) -def dump_databases(databases, log_prefix, location_config, dry_run): +def dump_databases(databases, config, log_prefix, dry_run): ''' Dump the given MongoDB databases to a named pipe. The databases are supplied as a sequence of - dicts, one dict describing each database as per the configuration schema. Use the given log - prefix in any log entries. Use the given location configuration dict to construct the - destination path. + dicts, one dict describing each database as per the configuration schema. Use the configuration + dict to construct the destination path and the given log prefix in any log entries. Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence. @@ -33,7 +32,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): for database in databases: name = database['name'] dump_filename = dump.make_database_dump_filename( - make_dump_path(location_config), name, database.get('hostname') + make_dump_path(config), name, database.get('hostname') ) dump_format = database.get('format', 'archive') @@ -82,35 +81,33 @@ def build_dump_command(database, dump_filename, dump_format): return command -def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover +def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover ''' Remove all database dump files for this hook regardless of the given databases. Use the log - prefix in any log entries. Use the given location configuration dict to construct the - destination path. If this is a dry run, then don't actually remove anything. + prefix in any log entries. Use the given configuration dict to construct the destination path. + If this is a dry run, then don't actually remove anything. ''' - dump.remove_database_dumps(make_dump_path(location_config), 'MongoDB', log_prefix, dry_run) + dump.remove_database_dumps(make_dump_path(config), 'MongoDB', log_prefix, dry_run) -def make_database_dump_pattern( - databases, log_prefix, location_config, name=None -): # pragma: no cover +def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover ''' - Given a sequence of configurations dicts, a prefix to log with, a location configuration dict, + Given a sequence of database configurations dicts, a configuration dict, a prefix to log with, and a database name to match, return the corresponding glob patterns to match the database dump in an archive. ''' - return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') + return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*') def restore_database_dump( - database_config, log_prefix, location_config, dry_run, extract_process, connection_params + database_config, config, log_prefix, dry_run, extract_process, connection_params ): ''' Restore the given MongoDB database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. - Use the given log prefix in any log entries. If this is a dry run, then don't actually restore - anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce - output to consume. + Use the configuration dict to construct the destination path and the given log prefix in any log + entries. If this is a dry run, then don't actually restore anything. Trigger the given active + extract process (an instance of subprocess.Popen) to produce output to consume. If the extract process is None, then restore the dump from the filesystem rather than from an extract stream. @@ -122,7 +119,7 @@ def restore_database_dump( database = database_config[0] dump_filename = dump.make_database_dump_filename( - make_dump_path(location_config), database['name'], database.get('hostname') + make_dump_path(config), database['name'], database.get('hostname') ) restore_command = build_restore_command( extract_process, database, dump_filename, connection_params diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index aee13d80..8ca2f6ba 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -12,13 +12,11 @@ from borgmatic.hooks import dump logger = logging.getLogger(__name__) -def make_dump_path(location_config): # pragma: no cover +def make_dump_path(config): # pragma: no cover ''' - Make the dump path from the given location configuration and the name of this hook. + Make the dump path from the given configuration dict and the name of this hook. ''' - return dump.make_database_dump_path( - location_config.get('borgmatic_source_directory'), 'mysql_databases' - ) + return dump.make_database_dump_path(config.get('borgmatic_source_directory'), 'mysql_databases') SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') @@ -106,12 +104,11 @@ def execute_dump_command( ) -def dump_databases(databases, log_prefix, location_config, dry_run): +def dump_databases(databases, config, log_prefix, dry_run): ''' Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence - of dicts, one dict describing each database as per the configuration schema. Use the given log - prefix in any log entries. Use the given location configuration dict to construct the - destination path. + of dicts, one dict describing each database as per the configuration schema. Use the given + configuration dict to construct the destination path and the given log prefix in any log entries. Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence. @@ -122,7 +119,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): logger.info(f'{log_prefix}: Dumping MySQL databases{dry_run_label}') for database in databases: - dump_path = make_dump_path(location_config) + dump_path = make_dump_path(config) extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None dump_database_names = database_names_to_dump( database, extra_environment, log_prefix, dry_run @@ -165,28 +162,26 @@ def dump_databases(databases, log_prefix, location_config, dry_run): return [process for process in processes if process] -def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover +def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover ''' - Remove all database dump files for this hook regardless of the given databases. Use the log - prefix in any log entries. Use the given location configuration dict to construct the - destination path. If this is a dry run, then don't actually remove anything. + Remove all database dump files for this hook regardless of the given databases. Use the given + configuration dict to construct the destination path and the log prefix in any log entries. If + this is a dry run, then don't actually remove anything. ''' - dump.remove_database_dumps(make_dump_path(location_config), 'MySQL', log_prefix, dry_run) + dump.remove_database_dumps(make_dump_path(config), 'MySQL', log_prefix, dry_run) -def make_database_dump_pattern( - databases, log_prefix, location_config, name=None -): # pragma: no cover +def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover ''' - Given a sequence of configurations dicts, a prefix to log with, a location configuration dict, - and a database name to match, return the corresponding glob patterns to match the database dump - in an archive. + Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a + database name to match, return the corresponding glob patterns to match the database dump in an + archive. ''' - return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') + return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*') def restore_database_dump( - database_config, log_prefix, location_config, dry_run, extract_process, connection_params + database_config, config, log_prefix, dry_run, extract_process, connection_params ): ''' Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a diff --git a/borgmatic/hooks/ntfy.py b/borgmatic/hooks/ntfy.py index 8a6f0fb8..50aa387a 100644 --- a/borgmatic/hooks/ntfy.py +++ b/borgmatic/hooks/ntfy.py @@ -14,7 +14,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): ''' Ping the configured Ntfy topic. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. diff --git a/borgmatic/hooks/pagerduty.py b/borgmatic/hooks/pagerduty.py index 561b1e24..382a402f 100644 --- a/borgmatic/hooks/pagerduty.py +++ b/borgmatic/hooks/pagerduty.py @@ -21,7 +21,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): ''' If this is an error state, create a PagerDuty event with the configured integration key. Use the given configuration filename in any log entries. If this is a dry run, then don't actually diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index ecb5f3c3..7bef5a70 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -14,12 +14,12 @@ from borgmatic.hooks import dump logger = logging.getLogger(__name__) -def make_dump_path(location_config): # pragma: no cover +def make_dump_path(config): # pragma: no cover ''' - Make the dump path from the given location configuration and the name of this hook. + Make the dump path from the given configuration dict and the name of this hook. ''' return dump.make_database_dump_path( - location_config.get('borgmatic_source_directory'), 'postgresql_databases' + config.get('borgmatic_source_directory'), 'postgresql_databases' ) @@ -92,12 +92,12 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): ) -def dump_databases(databases, log_prefix, location_config, dry_run): +def dump_databases(databases, config, log_prefix, dry_run): ''' Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of - dicts, one dict describing each database as per the configuration schema. Use the given log - prefix in any log entries. Use the given location configuration dict to construct the - destination path. + dicts, one dict describing each database as per the configuration schema. Use the given + configuration dict to construct the destination path and the given log prefix in any log + entries. Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence. @@ -111,7 +111,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): for database in databases: extra_environment = make_extra_environment(database) - dump_path = make_dump_path(location_config) + dump_path = make_dump_path(config) dump_database_names = database_names_to_dump( database, extra_environment, log_prefix, dry_run ) @@ -183,35 +183,33 @@ def dump_databases(databases, log_prefix, location_config, dry_run): return processes -def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover +def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover ''' - Remove all database dump files for this hook regardless of the given databases. Use the log - prefix in any log entries. Use the given location configuration dict to construct the - destination path. If this is a dry run, then don't actually remove anything. + Remove all database dump files for this hook regardless of the given databases. Use the given + configuration dict to construct the destination path and the log prefix in any log entries. If + this is a dry run, then don't actually remove anything. ''' - dump.remove_database_dumps(make_dump_path(location_config), 'PostgreSQL', log_prefix, dry_run) + dump.remove_database_dumps(make_dump_path(config), 'PostgreSQL', log_prefix, dry_run) -def make_database_dump_pattern( - databases, log_prefix, location_config, name=None -): # pragma: no cover +def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover ''' - Given a sequence of configurations dicts, a prefix to log with, a location configuration dict, - and a database name to match, return the corresponding glob patterns to match the database dump - in an archive. + Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a + database name to match, return the corresponding glob patterns to match the database dump in an + archive. ''' - return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*') + return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*') def restore_database_dump( - database_config, log_prefix, location_config, dry_run, extract_process, connection_params + database_config, config, log_prefix, dry_run, extract_process, connection_params ): ''' Restore the given PostgreSQL database from an extract stream. The database is supplied as a one-element sequence containing a dict describing the database, as per the configuration schema. - Use the given log prefix in any log entries. If this is a dry run, then don't actually restore - anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce - output to consume. + Use the given configuration dict to construct the destination path and the given log prefix in + any log entries. If this is a dry run, then don't actually restore anything. Trigger the given + active extract process (an instance of subprocess.Popen) to produce output to consume. If the extract process is None, then restore the dump from the filesystem rather than from an extract stream. @@ -236,7 +234,7 @@ def restore_database_dump( all_databases = bool(database['name'] == 'all') dump_filename = dump.make_database_dump_filename( - make_dump_path(location_config), database['name'], database.get('hostname') + make_dump_path(config), database['name'], database.get('hostname') ) psql_command = shlex.split(database.get('psql_command') or 'psql') analyze_command = ( diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py index 21b1455a..109f253a 100644 --- a/borgmatic/hooks/sqlite.py +++ b/borgmatic/hooks/sqlite.py @@ -7,21 +7,21 @@ from borgmatic.hooks import dump logger = logging.getLogger(__name__) -def make_dump_path(location_config): # pragma: no cover +def make_dump_path(config): # pragma: no cover ''' - Make the dump path from the given location configuration and the name of this hook. + Make the dump path from the given configuration dict and the name of this hook. ''' return dump.make_database_dump_path( - location_config.get('borgmatic_source_directory'), 'sqlite_databases' + config.get('borgmatic_source_directory'), 'sqlite_databases' ) -def dump_databases(databases, log_prefix, location_config, dry_run): +def dump_databases(databases, config, log_prefix, dry_run): ''' Dump the given SQLite3 databases to a file. The databases are supplied as a sequence of - configuration dicts, as per the configuration schema. Use the given log prefix in any log - entries. Use the given location configuration dict to construct the destination path. If this - is a dry run, then don't actually dump anything. + configuration dicts, as per the configuration schema. Use the given configuration dict to + construct the destination path and the given log prefix in any log entries. If this is a dry + run, then don't actually dump anything. ''' dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] @@ -38,7 +38,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): f'{log_prefix}: No SQLite database at {database_path}; An empty database will be created and dumped' ) - dump_path = make_dump_path(location_config) + dump_path = make_dump_path(config) dump_filename = dump.make_database_dump_filename(dump_path, database['name']) if os.path.exists(dump_filename): logger.warning( @@ -65,28 +65,26 @@ def dump_databases(databases, log_prefix, location_config, dry_run): return processes -def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover +def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover ''' Remove the given SQLite3 database dumps from the filesystem. The databases are supplied as a - sequence of configuration dicts, as per the configuration schema. Use the given log prefix in - any log entries. Use the given location configuration dict to construct the destination path. - If this is a dry run, then don't actually remove anything. + sequence of configuration dicts, as per the configuration schema. Use the given configuration + dict to construct the destination path and the given log prefix in any log entries. If this is a + dry run, then don't actually remove anything. ''' - dump.remove_database_dumps(make_dump_path(location_config), 'SQLite', log_prefix, dry_run) + dump.remove_database_dumps(make_dump_path(config), 'SQLite', log_prefix, dry_run) -def make_database_dump_pattern( - databases, log_prefix, location_config, name=None -): # pragma: no cover +def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover ''' Make a pattern that matches the given SQLite3 databases. The databases are supplied as a sequence of configuration dicts, as per the configuration schema. ''' - return dump.make_database_dump_filename(make_dump_path(location_config), name) + return dump.make_database_dump_filename(make_dump_path(config), name) def restore_database_dump( - database_config, log_prefix, location_config, dry_run, extract_process, connection_params + database_config, config, log_prefix, dry_run, extract_process, connection_params ): ''' Restore the given SQLite3 database from an extract stream. The database is supplied as a diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index a76d43f1..9a121215 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -84,7 +84,6 @@ def test_prune_archives_command_does_not_duplicate_flags_or_raise(): False, 'repo', {}, - {}, '2.3.4', fuzz_argument(arguments, argument_name), argparse.Namespace(log_json=False), diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index cf4b3945..5df1825a 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -13,43 +13,43 @@ def test_insert_newline_before_comment_does_not_raise(): config = module.yaml.comments.CommentedMap([(field_name, 33)]) config.yaml_set_comment_before_after_key(key=field_name, before='Comment') - module._insert_newline_before_comment(config, field_name) + module.insert_newline_before_comment(config, field_name) def test_comment_out_line_skips_blank_line(): line = ' \n' - assert module._comment_out_line(line) == line + assert module.comment_out_line(line) == line def test_comment_out_line_skips_already_commented_out_line(): line = ' # foo' - assert module._comment_out_line(line) == line + assert module.comment_out_line(line) == line def test_comment_out_line_comments_section_name(): line = 'figgy-pudding:' - assert module._comment_out_line(line) == '# ' + line + assert module.comment_out_line(line) == '# ' + line def test_comment_out_line_comments_indented_option(): line = ' enabled: true' - assert module._comment_out_line(line) == ' # enabled: true' + assert module.comment_out_line(line) == ' # enabled: true' def test_comment_out_line_comments_twice_indented_option(): line = ' - item' - assert module._comment_out_line(line) == ' # - item' + assert module.comment_out_line(line) == ' # - item' def test_comment_out_optional_configuration_comments_optional_config_only(): # The "# COMMENT_OUT" comment is a sentinel used to express that the following key is optional. # It's stripped out of the final output. - flexmock(module)._comment_out_line = lambda line: '# ' + line + flexmock(module).comment_out_line = lambda line: '# ' + line config = ''' # COMMENT_OUT foo: @@ -84,7 +84,7 @@ location: # other: thing ''' - assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip() + assert module.comment_out_optional_configuration(config.strip()) == expected_config.strip() def test_render_configuration_converts_configuration_to_yaml_string(): @@ -204,10 +204,10 @@ def test_generate_sample_configuration_does_not_raise(): builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('schema.yaml').and_return('') flexmock(module.yaml).should_receive('round_trip_load') - flexmock(module).should_receive('_schema_to_sample_configuration') + flexmock(module).should_receive('schema_to_sample_configuration') flexmock(module).should_receive('merge_source_configuration_into_destination') flexmock(module).should_receive('render_configuration') - flexmock(module).should_receive('_comment_out_optional_configuration') + flexmock(module).should_receive('comment_out_optional_configuration') flexmock(module).should_receive('write_configuration') module.generate_sample_configuration(False, None, 'dest.yaml', 'schema.yaml') @@ -219,10 +219,10 @@ def test_generate_sample_configuration_with_source_filename_does_not_raise(): flexmock(module.yaml).should_receive('round_trip_load') flexmock(module.load).should_receive('load_configuration') flexmock(module.normalize).should_receive('normalize') - flexmock(module).should_receive('_schema_to_sample_configuration') + flexmock(module).should_receive('schema_to_sample_configuration') flexmock(module).should_receive('merge_source_configuration_into_destination') flexmock(module).should_receive('render_configuration') - flexmock(module).should_receive('_comment_out_optional_configuration') + flexmock(module).should_receive('comment_out_optional_configuration') flexmock(module).should_receive('write_configuration') module.generate_sample_configuration(False, 'source.yaml', 'dest.yaml', 'schema.yaml') @@ -232,10 +232,10 @@ def test_generate_sample_configuration_with_dry_run_does_not_write_file(): builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('schema.yaml').and_return('') flexmock(module.yaml).should_receive('round_trip_load') - flexmock(module).should_receive('_schema_to_sample_configuration') + flexmock(module).should_receive('schema_to_sample_configuration') flexmock(module).should_receive('merge_source_configuration_into_destination') flexmock(module).should_receive('render_configuration') - flexmock(module).should_receive('_comment_out_optional_configuration') + flexmock(module).should_receive('comment_out_optional_configuration') flexmock(module).should_receive('write_configuration').never() module.generate_sample_configuration(True, None, 'dest.yaml', 'schema.yaml') diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 1abc3ba5..d446421e 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -40,35 +40,32 @@ def mock_config_and_schema(config_yaml, schema_yaml=None): def test_parse_configuration_transforms_file_into_mapping(): mock_config_and_schema( ''' - location: - source_directories: - - /home - - /etc + source_directories: + - /home + - /etc - repositories: - - path: hostname.borg + repositories: + - path: hostname.borg - retention: - keep_minutely: 60 - keep_hourly: 24 - keep_daily: 7 + keep_minutely: 60 + keep_hourly: 24 + keep_daily: 7 - consistency: - checks: - - name: repository - - name: archives + checks: + - name: repository + - name: archives ''' ) config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': { - 'source_directories': ['/home', '/etc'], - 'repositories': [{'path': 'hostname.borg'}], - }, - 'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60}, - 'consistency': {'checks': [{'name': 'repository'}, {'name': 'archives'}]}, + 'source_directories': ['/home', '/etc'], + 'repositories': [{'path': 'hostname.borg'}], + 'keep_daily': 7, + 'keep_hourly': 24, + 'keep_minutely': 60, + 'checks': [{'name': 'repository'}, {'name': 'archives'}], } assert logs == [] @@ -78,22 +75,19 @@ def test_parse_configuration_passes_through_quoted_punctuation(): mock_config_and_schema( f''' - location: - source_directories: - - "/home/{escaped_punctuation}" + source_directories: + - "/home/{escaped_punctuation}" - repositories: - - path: test.borg + repositories: + - path: test.borg ''' ) config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': { - 'source_directories': [f'/home/{string.punctuation}'], - 'repositories': [{'path': 'test.borg'}], - } + 'source_directories': [f'/home/{string.punctuation}'], + 'repositories': [{'path': 'test.borg'}], } assert logs == [] @@ -101,26 +95,22 @@ def test_parse_configuration_passes_through_quoted_punctuation(): def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): mock_config_and_schema( ''' - location: - source_directories: - - /home + source_directories: + - /home - repositories: - - path: hostname.borg + repositories: + - path: hostname.borg ''', ''' map: - location: + source_directories: required: true - map: - source_directories: - required: true - seq: - - type: scalar - repositories: - required: true - seq: - - type: scalar + seq: + - type: scalar + repositories: + required: true + seq: + - type: scalar ''', ) @@ -130,12 +120,11 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): def test_parse_configuration_inlines_include(): mock_config_and_schema( ''' - location: - source_directories: - - /home + source_directories: + - /home - repositories: - - path: hostname.borg + repositories: + - path: hostname.borg retention: !include include.yaml @@ -154,25 +143,25 @@ def test_parse_configuration_inlines_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]}, - 'retention': {'keep_daily': 7, 'keep_hourly': 24}, + 'source_directories': ['/home'], + 'repositories': [{'path': 'hostname.borg'}], + 'keep_daily': 7, + 'keep_hourly': 24, } - assert logs == [] + assert len(logs) == 1 def test_parse_configuration_merges_include(): mock_config_and_schema( ''' - location: - source_directories: - - /home + source_directories: + - /home - repositories: - - path: hostname.borg + repositories: + - path: hostname.borg - retention: - keep_daily: 1 - <<: !include include.yaml + keep_daily: 1 + <<: !include include.yaml ''' ) builtins = flexmock(sys.modules['builtins']) @@ -188,8 +177,10 @@ def test_parse_configuration_merges_include(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]}, - 'retention': {'keep_daily': 1, 'keep_hourly': 24}, + 'source_directories': ['/home'], + 'repositories': [{'path': 'hostname.borg'}], + 'keep_daily': 1, + 'keep_hourly': 24, } assert logs == [] @@ -218,10 +209,9 @@ def test_parse_configuration_raises_for_syntax_error(): def test_parse_configuration_raises_for_validation_error(): mock_config_and_schema( ''' - location: - source_directories: yes - repositories: - - path: hostname.borg + source_directories: yes + repositories: + - path: hostname.borg ''' ) @@ -232,14 +222,13 @@ def test_parse_configuration_raises_for_validation_error(): def test_parse_configuration_applies_overrides(): mock_config_and_schema( ''' - location: - source_directories: - - /home + source_directories: + - /home - repositories: - - path: hostname.borg + repositories: + - path: hostname.borg - local_path: borg1 + local_path: borg1 ''' ) @@ -248,11 +237,9 @@ def test_parse_configuration_applies_overrides(): ) assert config == { - 'location': { - 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg'}], - 'local_path': 'borg2', - } + 'source_directories': ['/home'], + 'repositories': [{'path': 'hostname.borg'}], + 'local_path': 'borg2', } assert logs == [] @@ -274,10 +261,8 @@ def test_parse_configuration_applies_normalization(): config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { - 'location': { - 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg'}], - 'exclude_if_present': ['.nobackup'], - } + 'source_directories': ['/home'], + 'repositories': [{'path': 'hostname.borg'}], + 'exclude_if_present': ['.nobackup'], } assert logs diff --git a/tests/unit/actions/test_borg.py b/tests/unit/actions/test_borg.py index 2e03ec9c..6cabe0d9 100644 --- a/tests/unit/actions/test_borg.py +++ b/tests/unit/actions/test_borg.py @@ -14,7 +14,7 @@ def test_run_borg_does_not_raise(): module.run_borg( repository={'path': 'repos'}, - storage={}, + config={}, local_borg_version=None, global_arguments=flexmock(log_json=False), borg_arguments=borg_arguments, diff --git a/tests/unit/actions/test_break_lock.py b/tests/unit/actions/test_break_lock.py index 5949d7c1..edc60f60 100644 --- a/tests/unit/actions/test_break_lock.py +++ b/tests/unit/actions/test_break_lock.py @@ -11,7 +11,7 @@ def test_run_break_lock_does_not_raise(): module.run_break_lock( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, break_lock_arguments=break_lock_arguments, global_arguments=flexmock(), diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 05f63b6a..72798e02 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -5,9 +5,6 @@ from borgmatic.actions import check as module def test_run_check_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.checks).should_receive( - 'repository_enabled_for_checks' - ).and_return(True) flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.check).should_receive('check_archives').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) @@ -23,10 +20,7 @@ def test_run_check_calls_hooks_for_configured_repository(): module.run_check( config_filename='test.yaml', repository={'path': 'repo'}, - location={'repositories': ['repo']}, - storage={}, - consistency={}, - hooks={}, + config={'repositories': ['repo']}, hook_context={}, local_borg_version=None, check_arguments=check_arguments, @@ -54,10 +48,7 @@ def test_run_check_runs_with_selected_repository(): module.run_check( config_filename='test.yaml', repository={'path': 'repo'}, - location={'repositories': ['repo']}, - storage={}, - consistency={}, - hooks={}, + config={'repositories': ['repo']}, hook_context={}, local_borg_version=None, check_arguments=check_arguments, @@ -85,10 +76,7 @@ def test_run_check_bails_if_repository_does_not_match(): module.run_check( config_filename='test.yaml', repository={'path': 'repo'}, - location={'repositories': ['repo']}, - storage={}, - consistency={}, - hooks={}, + config={'repositories': ['repo']}, hook_context={}, local_borg_version=None, check_arguments=check_arguments, diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index fbd4f905..0df83fdd 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -17,9 +17,7 @@ def test_compact_actions_calls_hooks_for_configured_repository(): module.run_compact( config_filename='test.yaml', repository={'path': 'repo'}, - storage={}, - retention={}, - hooks={}, + config={}, hook_context={}, local_borg_version=None, compact_arguments=compact_arguments, @@ -45,9 +43,7 @@ def test_compact_runs_with_selected_repository(): module.run_compact( config_filename='test.yaml', repository={'path': 'repo'}, - storage={}, - retention={}, - hooks={}, + config={}, hook_context={}, local_borg_version=None, compact_arguments=compact_arguments, @@ -73,9 +69,7 @@ def test_compact_bails_if_repository_does_not_match(): module.run_compact( config_filename='test.yaml', repository={'path': 'repo'}, - storage={}, - retention={}, - hooks={}, + config={}, hook_context={}, local_borg_version=None, compact_arguments=compact_arguments, diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index de94fd7e..355e544a 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -28,9 +28,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): module.run_create( config_filename='test.yaml', repository={'path': 'repo'}, - location={}, - storage={}, - hooks={}, + config={}, hook_context={}, local_borg_version=None, create_arguments=create_arguments, @@ -49,6 +47,11 @@ def test_run_create_runs_with_selected_repository(): ).once().and_return(True) flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() flexmock(module).should_receive('create_borgmatic_manifest').once() + flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).and_return({}) create_arguments = flexmock( repository=flexmock(), progress=flexmock(), @@ -62,9 +65,7 @@ def test_run_create_runs_with_selected_repository(): module.run_create( config_filename='test.yaml', repository={'path': 'repo'}, - location={}, - storage={}, - hooks={}, + config={}, hook_context={}, local_borg_version=None, create_arguments=create_arguments, @@ -96,9 +97,7 @@ def test_run_create_bails_if_repository_does_not_match(): module.run_create( config_filename='test.yaml', repository='repo', - location={}, - storage={}, - hooks={}, + config={}, hook_context={}, local_borg_version=None, create_arguments=create_arguments, diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index 6741d427..aea54af3 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -20,7 +20,7 @@ def test_run_export_tar_does_not_raise(): module.run_export_tar( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, export_tar_arguments=export_tar_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index 32b93b4e..7fadf4d7 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -21,9 +21,7 @@ def test_run_extract_calls_hooks(): module.run_extract( config_filename='test.yaml', repository={'path': 'repo'}, - location={'repositories': ['repo']}, - storage={}, - hooks={}, + config={'repositories': ['repo']}, hook_context={}, local_borg_version=None, extract_arguments=extract_arguments, diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py index 748d866f..1a5f5533 100644 --- a/tests/unit/actions/test_info.py +++ b/tests/unit/actions/test_info.py @@ -18,7 +18,7 @@ def test_run_info_does_not_raise(): list( module.run_info( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, info_arguments=info_arguments, global_arguments=flexmock(log_json=False), diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py index 07a1a58c..dd3b1326 100644 --- a/tests/unit/actions/test_list.py +++ b/tests/unit/actions/test_list.py @@ -18,7 +18,7 @@ def test_run_list_does_not_raise(): list( module.run_list( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, list_arguments=list_arguments, global_arguments=flexmock(log_json=False), diff --git a/tests/unit/actions/test_mount.py b/tests/unit/actions/test_mount.py index 743747d2..46607690 100644 --- a/tests/unit/actions/test_mount.py +++ b/tests/unit/actions/test_mount.py @@ -18,7 +18,7 @@ def test_run_mount_does_not_raise(): module.run_mount( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, mount_arguments=mount_arguments, global_arguments=flexmock(log_json=False), diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index 7af7ea77..d5dd182e 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -14,9 +14,7 @@ def test_run_prune_calls_hooks_for_configured_repository(): module.run_prune( config_filename='test.yaml', repository={'path': 'repo'}, - storage={}, - retention={}, - hooks={}, + config={}, hook_context={}, local_borg_version=None, prune_arguments=prune_arguments, @@ -39,9 +37,7 @@ def test_run_prune_runs_with_selected_repository(): module.run_prune( config_filename='test.yaml', repository={'path': 'repo'}, - storage={}, - retention={}, - hooks={}, + config={}, hook_context={}, local_borg_version=None, prune_arguments=prune_arguments, @@ -64,9 +60,7 @@ def test_run_prune_bails_if_repository_does_not_match(): module.run_prune( config_filename='test.yaml', repository='repo', - storage={}, - retention={}, - hooks={}, + config={}, hook_context={}, local_borg_version=None, prune_arguments=prune_arguments, diff --git a/tests/unit/actions/test_rcreate.py b/tests/unit/actions/test_rcreate.py index b77fa757..0de8d5df 100644 --- a/tests/unit/actions/test_rcreate.py +++ b/tests/unit/actions/test_rcreate.py @@ -19,7 +19,7 @@ def test_run_rcreate_does_not_raise(): module.run_rcreate( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, rcreate_arguments=arguments, global_arguments=flexmock(dry_run=False), @@ -46,7 +46,7 @@ def test_run_rcreate_bails_if_repository_does_not_match(): module.run_rcreate( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, rcreate_arguments=arguments, global_arguments=flexmock(dry_run=False), diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 4e19964f..35488808 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -6,7 +6,7 @@ import borgmatic.actions.restore as module def test_get_configured_database_matches_database_by_name(): assert module.get_configured_database( - hooks={ + config={ 'other_databases': [{'name': 'other'}], 'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}], }, @@ -18,7 +18,7 @@ def test_get_configured_database_matches_database_by_name(): def test_get_configured_database_matches_nothing_when_database_name_not_configured(): assert module.get_configured_database( - hooks={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]}, + config={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]}, archive_database_names={'postgresql_databases': ['foo']}, hook_name='postgresql_databases', database_name='quux', @@ -27,7 +27,7 @@ def test_get_configured_database_matches_nothing_when_database_name_not_configur def test_get_configured_database_matches_nothing_when_database_name_not_in_archive(): assert module.get_configured_database( - hooks={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]}, + config={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]}, archive_database_names={'postgresql_databases': ['bar']}, hook_name='postgresql_databases', database_name='foo', @@ -36,7 +36,7 @@ def test_get_configured_database_matches_nothing_when_database_name_not_in_archi def test_get_configured_database_matches_database_by_configuration_database_name(): assert module.get_configured_database( - hooks={'postgresql_databases': [{'name': 'all'}, {'name': 'bar'}]}, + config={'postgresql_databases': [{'name': 'all'}, {'name': 'bar'}]}, archive_database_names={'postgresql_databases': ['foo']}, hook_name='postgresql_databases', database_name='foo', @@ -46,7 +46,7 @@ def test_get_configured_database_matches_database_by_configuration_database_name def test_get_configured_database_with_unspecified_hook_matches_database_by_name(): assert module.get_configured_database( - hooks={ + config={ 'other_databases': [{'name': 'other'}], 'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}], }, @@ -69,8 +69,7 @@ def test_collect_archive_database_names_parses_archive_paths(): archive_database_names = module.collect_archive_database_names( repository={'path': 'repo'}, archive='archive', - location={'borgmatic_source_directory': '.borgmatic'}, - storage=flexmock(), + config={'borgmatic_source_directory': '.borgmatic'}, local_borg_version=flexmock(), global_arguments=flexmock(log_json=False), local_path=flexmock(), @@ -95,8 +94,7 @@ def test_collect_archive_database_names_parses_directory_format_archive_paths(): archive_database_names = module.collect_archive_database_names( repository={'path': 'repo'}, archive='archive', - location={'borgmatic_source_directory': '.borgmatic'}, - storage=flexmock(), + config={'borgmatic_source_directory': '.borgmatic'}, local_borg_version=flexmock(), global_arguments=flexmock(log_json=False), local_path=flexmock(), @@ -117,8 +115,7 @@ def test_collect_archive_database_names_skips_bad_archive_paths(): archive_database_names = module.collect_archive_database_names( repository={'path': 'repo'}, archive='archive', - location={'borgmatic_source_directory': '.borgmatic'}, - storage=flexmock(), + config={'borgmatic_source_directory': '.borgmatic'}, local_borg_version=flexmock(), global_arguments=flexmock(log_json=False), local_path=flexmock(), @@ -231,9 +228,7 @@ def test_run_restore_restores_each_database(): ).and_return(('postgresql_databases', {'name': 'bar'})) flexmock(module).should_receive('restore_single_database').with_args( repository=object, - location=object, - storage=object, - hooks=object, + config=object, local_borg_version=object, global_arguments=object, local_path=object, @@ -245,9 +240,7 @@ def test_run_restore_restores_each_database(): ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, - location=object, - storage=object, - hooks=object, + config=object, local_borg_version=object, global_arguments=object, local_path=object, @@ -261,9 +254,7 @@ def test_run_restore_restores_each_database(): module.run_restore( repository={'path': 'repo'}, - location=flexmock(), - storage=flexmock(), - hooks=flexmock(), + config=flexmock(), local_borg_version=flexmock(), restore_arguments=flexmock( repository='repo', @@ -293,9 +284,7 @@ def test_run_restore_bails_for_non_matching_repository(): module.run_restore( repository={'path': 'repo'}, - location=flexmock(), - storage=flexmock(), - hooks=flexmock(), + config=flexmock(), local_borg_version=flexmock(), restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), global_arguments=flexmock(dry_run=False), @@ -317,19 +306,19 @@ def test_run_restore_restores_database_configured_with_all_name(): flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock()) flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names) flexmock(module).should_receive('get_configured_database').with_args( - hooks=object, + config=object, archive_database_names=object, hook_name='postgresql_databases', database_name='foo', ).and_return(('postgresql_databases', {'name': 'foo'})) flexmock(module).should_receive('get_configured_database').with_args( - hooks=object, + config=object, archive_database_names=object, hook_name='postgresql_databases', database_name='bar', ).and_return((None, None)) flexmock(module).should_receive('get_configured_database').with_args( - hooks=object, + config=object, archive_database_names=object, hook_name='postgresql_databases', database_name='bar', @@ -337,9 +326,7 @@ def test_run_restore_restores_database_configured_with_all_name(): ).and_return(('postgresql_databases', {'name': 'bar'})) flexmock(module).should_receive('restore_single_database').with_args( repository=object, - location=object, - storage=object, - hooks=object, + config=object, local_borg_version=object, global_arguments=object, local_path=object, @@ -351,9 +338,7 @@ def test_run_restore_restores_database_configured_with_all_name(): ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, - location=object, - storage=object, - hooks=object, + config=object, local_borg_version=object, global_arguments=object, local_path=object, @@ -367,9 +352,7 @@ def test_run_restore_restores_database_configured_with_all_name(): module.run_restore( repository={'path': 'repo'}, - location=flexmock(), - storage=flexmock(), - hooks=flexmock(), + config=flexmock(), local_borg_version=flexmock(), restore_arguments=flexmock( repository='repo', @@ -401,19 +384,19 @@ def test_run_restore_skips_missing_database(): flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock()) flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names) flexmock(module).should_receive('get_configured_database').with_args( - hooks=object, + config=object, archive_database_names=object, hook_name='postgresql_databases', database_name='foo', ).and_return(('postgresql_databases', {'name': 'foo'})) flexmock(module).should_receive('get_configured_database').with_args( - hooks=object, + config=object, archive_database_names=object, hook_name='postgresql_databases', database_name='bar', ).and_return((None, None)) flexmock(module).should_receive('get_configured_database').with_args( - hooks=object, + config=object, archive_database_names=object, hook_name='postgresql_databases', database_name='bar', @@ -421,9 +404,7 @@ def test_run_restore_skips_missing_database(): ).and_return((None, None)) flexmock(module).should_receive('restore_single_database').with_args( repository=object, - location=object, - storage=object, - hooks=object, + config=object, local_borg_version=object, global_arguments=object, local_path=object, @@ -435,9 +416,7 @@ def test_run_restore_skips_missing_database(): ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, - location=object, - storage=object, - hooks=object, + config=object, local_borg_version=object, global_arguments=object, local_path=object, @@ -451,9 +430,7 @@ def test_run_restore_skips_missing_database(): module.run_restore( repository={'path': 'repo'}, - location=flexmock(), - storage=flexmock(), - hooks=flexmock(), + config=flexmock(), local_borg_version=flexmock(), restore_arguments=flexmock( repository='repo', @@ -486,22 +463,20 @@ def test_run_restore_restores_databases_from_different_hooks(): flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock()) flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names) flexmock(module).should_receive('get_configured_database').with_args( - hooks=object, + config=object, archive_database_names=object, hook_name='postgresql_databases', database_name='foo', ).and_return(('postgresql_databases', {'name': 'foo'})) flexmock(module).should_receive('get_configured_database').with_args( - hooks=object, + config=object, archive_database_names=object, hook_name='mysql_databases', database_name='bar', ).and_return(('mysql_databases', {'name': 'bar'})) flexmock(module).should_receive('restore_single_database').with_args( repository=object, - location=object, - storage=object, - hooks=object, + config=object, local_borg_version=object, global_arguments=object, local_path=object, @@ -513,9 +488,7 @@ def test_run_restore_restores_databases_from_different_hooks(): ).once() flexmock(module).should_receive('restore_single_database').with_args( repository=object, - location=object, - storage=object, - hooks=object, + config=object, local_borg_version=object, global_arguments=object, local_path=object, @@ -529,9 +502,7 @@ def test_run_restore_restores_databases_from_different_hooks(): module.run_restore( repository={'path': 'repo'}, - location=flexmock(), - storage=flexmock(), - hooks=flexmock(), + config=flexmock(), local_borg_version=flexmock(), restore_arguments=flexmock( repository='repo', diff --git a/tests/unit/actions/test_rinfo.py b/tests/unit/actions/test_rinfo.py index 7b2371a3..4ba73c41 100644 --- a/tests/unit/actions/test_rinfo.py +++ b/tests/unit/actions/test_rinfo.py @@ -12,7 +12,7 @@ def test_run_rinfo_does_not_raise(): list( module.run_rinfo( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, rinfo_arguments=rinfo_arguments, global_arguments=flexmock(log_json=False), diff --git a/tests/unit/actions/test_rlist.py b/tests/unit/actions/test_rlist.py index 4a59dc30..84798a76 100644 --- a/tests/unit/actions/test_rlist.py +++ b/tests/unit/actions/test_rlist.py @@ -12,7 +12,7 @@ def test_run_rlist_does_not_raise(): list( module.run_rlist( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, rlist_arguments=rlist_arguments, global_arguments=flexmock(), diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py index 58d8a160..03d259be 100644 --- a/tests/unit/actions/test_transfer.py +++ b/tests/unit/actions/test_transfer.py @@ -11,7 +11,7 @@ def test_run_transfer_does_not_raise(): module.run_transfer( repository={'path': 'repo'}, - storage={}, + config={}, local_borg_version=None, transfer_arguments=transfer_arguments, global_arguments=global_arguments, diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 2d7e1750..f38ec0e3 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -22,7 +22,7 @@ def test_run_arbitrary_borg_calls_borg_with_flags(): module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['break-lock', '::'], ) @@ -44,7 +44,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['break-lock', '::'], ) @@ -66,7 +66,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['break-lock', '::'], ) @@ -75,7 +75,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - storage_config = {'lock_wait': 5} + config = {'lock_wait': 5} flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') ) @@ -90,7 +90,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): module.run_arbitrary_borg( repository_path='repo', - storage_config=storage_config, + config=config, local_borg_version='1.2.3', options=['break-lock', '::'], ) @@ -111,7 +111,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['break-lock', '::$ARCHIVE'], archive='archive', @@ -133,7 +133,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['break-lock', '::'], local_path='borg1', @@ -157,7 +157,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags() module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['break-lock', '::'], remote_path='borg1', @@ -179,7 +179,7 @@ def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['list', '--progress', '::'], ) @@ -200,7 +200,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['--', 'break-lock', '::'], ) @@ -221,7 +221,7 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=[], ) @@ -243,7 +243,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['key', 'export', '::'], ) @@ -265,7 +265,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla module.run_arbitrary_borg( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', options=['debug', 'dump-manifest', '::', 'path'], ) diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 3dc55672..ff26cab7 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -22,7 +22,7 @@ def test_break_lock_calls_borg_with_required_flags(): module.break_lock( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -34,7 +34,7 @@ def test_break_lock_calls_borg_with_remote_path_flags(): module.break_lock( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), remote_path='borg1', @@ -47,7 +47,7 @@ def test_break_lock_calls_borg_with_umask_flags(): module.break_lock( repository_path='repo', - storage_config={'umask': '0770'}, + config={'umask': '0770'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -59,7 +59,7 @@ def test_break_lock_calls_borg_with_log_json_flags(): module.break_lock( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), ) @@ -71,7 +71,7 @@ def test_break_lock_calls_borg_with_lock_wait_flags(): module.break_lock( repository_path='repo', - storage_config={'lock_wait': '5'}, + config={'lock_wait': '5'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -84,7 +84,7 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter(): module.break_lock( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -97,7 +97,7 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags(): module.break_lock( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 79201166..c524b47e 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -99,8 +99,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks(): flexmock(module).should_receive('probe_for_check_time').and_return(None) assert module.filter_checks_on_frequency( - location_config={}, - consistency_config={}, + config={}, borg_repository_id='repo', checks=('repository', 'archives'), force=False, @@ -110,8 +109,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks(): def test_filter_checks_on_frequency_retains_unconfigured_check(): assert module.filter_checks_on_frequency( - location_config={}, - consistency_config={}, + config={}, borg_repository_id='repo', checks=('data',), force=False, @@ -122,8 +120,7 @@ def test_filter_checks_on_frequency_retains_check_without_frequency(): flexmock(module).should_receive('parse_frequency').and_return(None) assert module.filter_checks_on_frequency( - location_config={}, - consistency_config={'checks': [{'name': 'archives'}]}, + config={'checks': [{'name': 'archives'}]}, borg_repository_id='repo', checks=('archives',), force=False, @@ -141,8 +138,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency(): ) assert module.filter_checks_on_frequency( - location_config={}, - consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, + config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, borg_repository_id='repo', checks=('archives',), force=False, @@ -158,8 +154,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file() flexmock(module).should_receive('probe_for_check_time').and_return(None) assert module.filter_checks_on_frequency( - location_config={}, - consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, + config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, borg_repository_id='repo', checks=('archives',), force=False, @@ -178,8 +173,7 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency(): assert ( module.filter_checks_on_frequency( - location_config={}, - consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, + config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, borg_repository_id='repo', checks=('archives',), force=False, @@ -191,8 +185,7 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency(): def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_force(): assert module.filter_checks_on_frequency( - location_config={}, - consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, + config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, borg_repository_id='repo', checks=('archives',), force=True, @@ -616,7 +609,7 @@ def test_upgrade_check_times_renames_stale_temporary_check_path(): def test_check_archives_with_progress_calls_borg_with_progress_parameter(): checks = ('repository',) - consistency_config = {'check_last': None} + config = {'check_last': None} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -639,9 +632,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), progress=True, @@ -650,7 +641,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): def test_check_archives_with_repair_calls_borg_with_repair_parameter(): checks = ('repository',) - consistency_config = {'check_last': None} + config = {'check_last': None} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -673,9 +664,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), repair=True, @@ -693,7 +682,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): ) def test_check_archives_calls_borg_with_parameters(checks): check_last = flexmock() - consistency_config = {'check_last': check_last} + config = {'check_last': check_last} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -710,9 +699,7 @@ def test_check_archives_calls_borg_with_parameters(checks): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -721,7 +708,7 @@ def test_check_archives_calls_borg_with_parameters(checks): def test_check_archives_with_json_error_raises(): checks = ('archives',) check_last = flexmock() - consistency_config = {'check_last': check_last} + config = {'check_last': check_last} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"unexpected": {"id": "repo"}}' ) @@ -734,9 +721,7 @@ def test_check_archives_with_json_error_raises(): with pytest.raises(ValueError): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -745,7 +730,7 @@ def test_check_archives_with_json_error_raises(): def test_check_archives_with_missing_json_keys_raises(): checks = ('archives',) check_last = flexmock() - consistency_config = {'check_last': check_last} + config = {'check_last': check_last} flexmock(module.rinfo).should_receive('display_repository_info').and_return('{invalid JSON') flexmock(module).should_receive('upgrade_check_times') flexmock(module).should_receive('parse_checks') @@ -756,9 +741,7 @@ def test_check_archives_with_missing_json_keys_raises(): with pytest.raises(ValueError): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -767,7 +750,7 @@ def test_check_archives_with_missing_json_keys_raises(): def test_check_archives_with_extract_check_calls_extract_only(): checks = ('extract',) check_last = flexmock() - consistency_config = {'check_last': check_last} + config = {'check_last': check_last} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -784,9 +767,7 @@ def test_check_archives_with_extract_check_calls_extract_only(): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -794,7 +775,7 @@ def test_check_archives_with_extract_check_calls_extract_only(): def test_check_archives_with_log_info_calls_borg_with_info_parameter(): checks = ('repository',) - consistency_config = {'check_last': None} + config = {'check_last': None} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -812,9 +793,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -822,7 +801,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): checks = ('repository',) - consistency_config = {'check_last': None} + config = {'check_last': None} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -840,16 +819,14 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) def test_check_archives_without_any_checks_bails(): - consistency_config = {'check_last': None} + config = {'check_last': None} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -862,9 +839,7 @@ def test_check_archives_without_any_checks_bails(): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -873,7 +848,7 @@ def test_check_archives_without_any_checks_bails(): def test_check_archives_with_local_path_calls_borg_via_local_path(): checks = ('repository',) check_last = flexmock() - consistency_config = {'check_last': check_last} + config = {'check_last': check_last} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -890,9 +865,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), local_path='borg1', @@ -902,7 +875,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(): checks = ('repository',) check_last = flexmock() - consistency_config = {'check_last': check_last} + config = {'check_last': check_last} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -919,9 +892,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), remote_path='borg1', @@ -931,8 +902,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): checks = ('repository',) check_last = flexmock() - storage_config = {} - consistency_config = {'check_last': check_last} + config = {'check_last': check_last} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -949,9 +919,7 @@ def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): module.check_archives( repository_path='repo', - location_config={}, - storage_config=storage_config, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), ) @@ -960,8 +928,7 @@ def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): checks = ('repository',) check_last = flexmock() - storage_config = {'lock_wait': 5} - consistency_config = {'check_last': check_last} + config = {'lock_wait': 5, 'check_last': check_last} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -978,9 +945,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.check_archives( repository_path='repo', - location_config={}, - storage_config=storage_config, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -990,7 +955,7 @@ def test_check_archives_with_retention_prefix(): checks = ('repository',) check_last = flexmock() prefix = 'foo-' - consistency_config = {'check_last': check_last, 'prefix': prefix} + config = {'check_last': check_last, 'prefix': prefix} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -1007,9 +972,7 @@ def test_check_archives_with_retention_prefix(): module.check_archives( repository_path='repo', - location_config={}, - storage_config={}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -1017,7 +980,7 @@ def test_check_archives_with_retention_prefix(): def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): checks = ('repository',) - consistency_config = {'check_last': None} + config = {'check_last': None, 'extra_borg_options': {'check': '--extra --options'}} flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) @@ -1034,9 +997,7 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): module.check_archives( repository_path='repo', - location_config={}, - storage_config={'extra_borg_options': {'check': '--extra --options'}}, - consistency_config=consistency_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index beacf547..c8e3e7f4 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -27,7 +27,7 @@ def test_compact_segments_calls_borg_with_parameters(): module.compact_segments( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -40,7 +40,7 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): module.compact_segments( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), dry_run=False, @@ -54,7 +54,7 @@ def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): module.compact_segments( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), dry_run=False, @@ -66,7 +66,7 @@ def test_compact_segments_with_dry_run_skips_borg_call(): module.compact_segments( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), dry_run=True, @@ -80,7 +80,7 @@ def test_compact_segments_with_local_path_calls_borg_via_local_path(): module.compact_segments( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), local_path='borg1', @@ -94,7 +94,7 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter module.compact_segments( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), remote_path='borg1', @@ -108,7 +108,7 @@ def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): module.compact_segments( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), progress=True, @@ -122,7 +122,7 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p module.compact_segments( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), cleanup_commits=True, @@ -136,7 +136,7 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): module.compact_segments( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), threshold=20, @@ -144,14 +144,14 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): - storage_config = {'umask': '077'} + config = {'umask': '077'} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - storage_config=storage_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -164,21 +164,21 @@ def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): module.compact_segments( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), ) def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): - storage_config = {'lock_wait': 5} + config = {'lock_wait': 5} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - storage_config=storage_config, + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -191,7 +191,7 @@ def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options( module.compact_segments( dry_run=False, repository_path='repo', - storage_config={'extra_borg_options': {'compact': '--extra --options'}}, + config={'extra_borg_options': {'compact': '--extra --options'}}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 33e95607..5b196d9b 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -170,79 +170,75 @@ def test_ensure_files_readable_opens_filenames(filename_lists, opened_filenames) def test_make_pattern_flags_includes_pattern_filename_when_given(): pattern_flags = module.make_pattern_flags( - location_config={'patterns': ['R /', '- /var']}, pattern_filename='/tmp/patterns' + config={'patterns': ['R /', '- /var']}, pattern_filename='/tmp/patterns' ) assert pattern_flags == ('--patterns-from', '/tmp/patterns') def test_make_pattern_flags_includes_patterns_from_filenames_when_in_config(): - pattern_flags = module.make_pattern_flags( - location_config={'patterns_from': ['patterns', 'other']} - ) + pattern_flags = module.make_pattern_flags(config={'patterns_from': ['patterns', 'other']}) assert pattern_flags == ('--patterns-from', 'patterns', '--patterns-from', 'other') def test_make_pattern_flags_includes_both_filenames_when_patterns_given_and_patterns_from_in_config(): pattern_flags = module.make_pattern_flags( - location_config={'patterns_from': ['patterns']}, pattern_filename='/tmp/patterns' + config={'patterns_from': ['patterns']}, pattern_filename='/tmp/patterns' ) assert pattern_flags == ('--patterns-from', 'patterns', '--patterns-from', '/tmp/patterns') def test_make_pattern_flags_considers_none_patterns_from_filenames_as_empty(): - pattern_flags = module.make_pattern_flags(location_config={'patterns_from': None}) + pattern_flags = module.make_pattern_flags(config={'patterns_from': None}) assert pattern_flags == () def test_make_exclude_flags_includes_exclude_patterns_filename_when_given(): exclude_flags = module.make_exclude_flags( - location_config={'exclude_patterns': ['*.pyc', '/var']}, exclude_filename='/tmp/excludes' + config={'exclude_patterns': ['*.pyc', '/var']}, exclude_filename='/tmp/excludes' ) assert exclude_flags == ('--exclude-from', '/tmp/excludes') def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config(): - exclude_flags = module.make_exclude_flags( - location_config={'exclude_from': ['excludes', 'other']} - ) + exclude_flags = module.make_exclude_flags(config={'exclude_from': ['excludes', 'other']}) assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', 'other') def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config(): exclude_flags = module.make_exclude_flags( - location_config={'exclude_from': ['excludes']}, exclude_filename='/tmp/excludes' + config={'exclude_from': ['excludes']}, exclude_filename='/tmp/excludes' ) assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes') def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty(): - exclude_flags = module.make_exclude_flags(location_config={'exclude_from': None}) + exclude_flags = module.make_exclude_flags(config={'exclude_from': None}) assert exclude_flags == () def test_make_exclude_flags_includes_exclude_caches_when_true_in_config(): - exclude_flags = module.make_exclude_flags(location_config={'exclude_caches': True}) + exclude_flags = module.make_exclude_flags(config={'exclude_caches': True}) assert exclude_flags == ('--exclude-caches',) def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config(): - exclude_flags = module.make_exclude_flags(location_config={'exclude_caches': False}) + exclude_flags = module.make_exclude_flags(config={'exclude_caches': False}) assert exclude_flags == () def test_make_exclude_flags_includes_exclude_if_present_when_in_config(): exclude_flags = module.make_exclude_flags( - location_config={'exclude_if_present': ['exclude_me', 'also_me']} + config={'exclude_if_present': ['exclude_me', 'also_me']} ) assert exclude_flags == ( @@ -254,31 +250,31 @@ def test_make_exclude_flags_includes_exclude_if_present_when_in_config(): def test_make_exclude_flags_includes_keep_exclude_tags_when_true_in_config(): - exclude_flags = module.make_exclude_flags(location_config={'keep_exclude_tags': True}) + exclude_flags = module.make_exclude_flags(config={'keep_exclude_tags': True}) assert exclude_flags == ('--keep-exclude-tags',) def test_make_exclude_flags_does_not_include_keep_exclude_tags_when_false_in_config(): - exclude_flags = module.make_exclude_flags(location_config={'keep_exclude_tags': False}) + exclude_flags = module.make_exclude_flags(config={'keep_exclude_tags': False}) assert exclude_flags == () def test_make_exclude_flags_includes_exclude_nodump_when_true_in_config(): - exclude_flags = module.make_exclude_flags(location_config={'exclude_nodump': True}) + exclude_flags = module.make_exclude_flags(config={'exclude_nodump': True}) assert exclude_flags == ('--exclude-nodump',) def test_make_exclude_flags_does_not_include_exclude_nodump_when_false_in_config(): - exclude_flags = module.make_exclude_flags(location_config={'exclude_nodump': False}) + exclude_flags = module.make_exclude_flags(config={'exclude_nodump': False}) assert exclude_flags == () def test_make_exclude_flags_is_empty_when_config_has_no_excludes(): - exclude_flags = module.make_exclude_flags(location_config={}) + exclude_flags = module.make_exclude_flags(config={}) assert exclude_flags == () @@ -504,12 +500,11 @@ def test_create_archive_calls_borg_with_parameters(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -548,12 +543,11 @@ def test_create_archive_calls_borg_with_environment(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -594,12 +588,11 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'patterns': ['pattern'], }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -644,11 +637,10 @@ def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sourc module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']), ) @@ -689,12 +681,11 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': ['exclude'], }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -733,12 +724,11 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -774,12 +764,11 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, @@ -819,12 +808,11 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -860,12 +848,11 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, @@ -904,12 +891,11 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): module.create_archive( dry_run=True, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -950,12 +936,11 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete module.create_archive( dry_run=True, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), stats=True, @@ -994,12 +979,12 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'checkpoint_interval': 600, }, - storage_config={'checkpoint_interval': 600}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1037,12 +1022,12 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'checkpoint_volume': 1024, }, - storage_config={'checkpoint_volume': 1024}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1080,12 +1065,12 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'chunker_params': '1,2,3,4', }, - storage_config={'chunker_params': '1,2,3,4'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1123,12 +1108,12 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'compression': 'rle', }, - storage_config={'compression': 'rle'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1172,12 +1157,12 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'upload_rate_limit': 100, }, - storage_config={'upload_rate_limit': 100}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1217,13 +1202,12 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'working_directory': '/working/dir', 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1261,13 +1245,12 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'one_file_system': True, 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1311,13 +1294,12 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'numeric_ids': True, 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1365,13 +1347,12 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'read_special': True, 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1421,13 +1402,12 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], option_name: option_value, 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1476,13 +1456,12 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'atime': option_value, 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1531,13 +1510,12 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'flags': option_value, 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1575,13 +1553,12 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'files_cache': 'ctime,size', 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1619,12 +1596,11 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), local_path='borg1', @@ -1663,12 +1639,11 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), remote_path='borg1', @@ -1707,12 +1682,12 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'umask': 740, }, - storage_config={'umask': 740}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1750,12 +1725,11 @@ def test_create_archive_with_log_json_calls_borg_with_log_json_parameters(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True, used_config_paths=[]), ) @@ -1793,12 +1767,12 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'lock_wait': 5, }, - storage_config={'lock_wait': 5}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1836,12 +1810,11 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), stats=True, @@ -1880,12 +1853,11 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), list_files=True, @@ -1930,12 +1902,11 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), progress=True, @@ -1974,12 +1945,11 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), progress=True, @@ -2035,12 +2005,11 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), progress=True, @@ -2099,13 +2068,12 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, 'read_special': False, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, @@ -2168,12 +2136,11 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, @@ -2232,13 +2199,12 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, 'read_special': True, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, @@ -2274,12 +2240,11 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): json_output = module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, @@ -2317,12 +2282,11 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() json_output = module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, @@ -2365,12 +2329,11 @@ def test_create_archive_with_source_directories_glob_expands(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo*'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2409,12 +2372,11 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo*'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2452,12 +2414,11 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo*'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2495,12 +2456,12 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'archive_name_format': 'ARCHIVE_NAME', }, - storage_config={'archive_name_format': 'ARCHIVE_NAME'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2539,12 +2500,12 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'archive_name_format': 'Documents_{hostname}-{now}', # noqa: FS003 }, - storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2583,12 +2544,12 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): module.create_archive( dry_run=False, repository_path='{fqdn}', # noqa: FS003 - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['{fqdn}'], # noqa: FS003 'exclude_patterns': None, + 'archive_name_format': 'Documents_{hostname}-{now}', # noqa: FS003 }, - storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2626,12 +2587,12 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'extra_borg_options': {'create': '--extra --options'}, }, - storage_config={'extra_borg_options': {'create': '--extra --options'}}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2687,12 +2648,11 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, @@ -2712,13 +2672,12 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ module.create_archive( dry_run=False, repository_path='repo', - location_config={ + config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, 'source_directories_must_exist': True, }, - storage_config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False, used_config_paths=[]), ) diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index 5fb7bff2..32b0967f 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -36,7 +36,7 @@ def test_export_tar_archive_calls_borg_with_path_parameters(): archive='archive', paths=['path1', 'path2'], destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -58,7 +58,7 @@ def test_export_tar_archive_calls_borg_with_local_path_parameters(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), local_path='borg1', @@ -81,7 +81,7 @@ def test_export_tar_archive_calls_borg_with_remote_path_parameters(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), remote_path='borg1', @@ -104,7 +104,7 @@ def test_export_tar_archive_calls_borg_with_umask_parameters(): archive='archive', paths=None, destination_path='test.tar', - storage_config={'umask': '0770'}, + config={'umask': '0770'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -124,7 +124,7 @@ def test_export_tar_archive_calls_borg_with_log_json_parameter(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), ) @@ -146,7 +146,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_parameters(): archive='archive', paths=None, destination_path='test.tar', - storage_config={'lock_wait': '5'}, + config={'lock_wait': '5'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -167,7 +167,7 @@ def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -190,7 +190,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_parameters(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -210,7 +210,7 @@ def test_export_tar_archive_calls_borg_with_dry_run_parameter(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -232,7 +232,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_parameters(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), tar_filter='bzip2', @@ -256,7 +256,7 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), list_files=True, @@ -279,7 +279,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), strip_components=5, @@ -300,7 +300,7 @@ def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): archive='archive', paths=None, destination_path='test.tar', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -320,7 +320,7 @@ def test_export_tar_archive_calls_borg_with_stdout_destination_path(): archive='archive', paths=None, destination_path='-', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index a4032f6c..a65aac73 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -25,7 +25,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_last_archive(): ) module.extract_last_archive_dry_run( - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), repository_path='repo', @@ -38,7 +38,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_not_raise(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(('repo',)) module.extract_last_archive_dry_run( - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), repository_path='repo', @@ -55,7 +55,7 @@ def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_paramet ) module.extract_last_archive_dry_run( - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), repository_path='repo', @@ -74,7 +74,7 @@ def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_param ) module.extract_last_archive_dry_run( - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), repository_path='repo', @@ -90,7 +90,7 @@ def test_extract_last_archive_dry_run_calls_borg_via_local_path(): ) module.extract_last_archive_dry_run( - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), repository_path='repo', @@ -109,7 +109,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_flags(): ) module.extract_last_archive_dry_run( - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), repository_path='repo', @@ -126,7 +126,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_log_json_flag(): ) module.extract_last_archive_dry_run( - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), repository_path='repo', @@ -144,7 +144,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_flags(): ) module.extract_last_archive_dry_run( - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), repository_path='repo', @@ -168,8 +168,7 @@ def test_extract_archive_calls_borg_with_path_flags(): repository='repo', archive='archive', paths=['path1', 'path2'], - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -191,8 +190,7 @@ def test_extract_archive_calls_borg_with_remote_path_flags(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), remote_path='borg1', @@ -222,8 +220,7 @@ def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available repository='repo', archive='archive', paths=None, - location_config={'numeric_ids': True}, - storage_config={}, + config={'numeric_ids': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -245,8 +242,7 @@ def test_extract_archive_calls_borg_with_umask_flags(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={'umask': '0770'}, + config={'umask': '0770'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -265,8 +261,7 @@ def test_extract_archive_calls_borg_with_log_json_flags(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), ) @@ -288,8 +283,7 @@ def test_extract_archive_calls_borg_with_lock_wait_flags(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={'lock_wait': '5'}, + config={'lock_wait': '5'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -312,8 +306,7 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -338,8 +331,7 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_flags(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -361,8 +353,7 @@ def test_extract_archive_calls_borg_with_dry_run_parameter(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -384,8 +375,7 @@ def test_extract_archive_calls_borg_with_destination_path(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), destination_path='/dest', @@ -408,8 +398,7 @@ def test_extract_archive_calls_borg_with_strip_components(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), strip_components=5, @@ -442,8 +431,7 @@ def test_extract_archive_calls_borg_with_strip_components_calculated_from_all(): repository='repo', archive='archive', paths=['foo/bar/baz.txt', 'foo/bar.txt'], - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), strip_components='all', @@ -467,8 +455,7 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), strip_components='all', @@ -497,8 +484,7 @@ def test_extract_archive_calls_borg_with_progress_parameter(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), progress=True, @@ -514,8 +500,7 @@ def test_extract_archive_with_progress_and_extract_to_stdout_raises(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), progress=True, @@ -548,8 +533,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process(): repository='repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), extract_to_stdout=True, @@ -579,8 +563,7 @@ def test_extract_archive_skips_abspath_for_remote_repository(): repository='server:repo', archive='archive', paths=None, - location_config={}, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index e7b126a6..3e93bb73 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -27,7 +27,7 @@ def test_display_archives_info_calls_borg_with_parameters(): module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), @@ -53,7 +53,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), @@ -78,7 +78,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu insert_logging_mock(logging.INFO) json_output = module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), @@ -107,7 +107,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), @@ -132,7 +132,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp insert_logging_mock(logging.DEBUG) json_output = module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), @@ -158,7 +158,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): json_output = module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), @@ -186,7 +186,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive='archive', json=False, prefix=None, match_archives=None), @@ -212,7 +212,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), @@ -242,7 +242,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), @@ -272,7 +272,7 @@ def test_display_archives_info_with_log_json_calls_borg_with_log_json_parameters module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=True), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), @@ -291,7 +291,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) - storage_config = {'lock_wait': 5} + config = {'lock_wait': 5} flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'info', '--lock-wait', '5', '--repo', 'repo'), @@ -302,7 +302,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete module.display_archives_info( repository_path='repo', - storage_config=storage_config, + config=config, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), @@ -331,7 +331,7 @@ def test_display_archives_info_transforms_prefix_into_match_archives_parameters( module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix='foo'), @@ -360,7 +360,7 @@ def test_display_archives_info_prefers_prefix_over_archive_name_format(): module.display_archives_info( repository_path='repo', - storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix='foo'), @@ -386,7 +386,7 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive module.display_archives_info( repository_path='repo', - storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), @@ -412,7 +412,7 @@ def test_display_archives_with_match_archives_option_calls_borg_with_match_archi module.display_archives_info( repository_path='repo', - storage_config={ + config={ 'archive_name_format': 'bar-{now}', # noqa: FS003 'match_archives': 'sh:foo-*', }, @@ -441,7 +441,7 @@ def test_display_archives_with_match_archives_flag_calls_borg_with_match_archive module.display_archives_info( repository_path='repo', - storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), @@ -471,7 +471,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock( @@ -523,7 +523,7 @@ def test_display_archives_info_with_date_based_matching_calls_borg_with_date_bas ) module.display_archives_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=info_arguments, diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 4e3a5f7c..2f82b802 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -17,7 +17,7 @@ def test_make_list_command_includes_log_info(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), global_arguments=flexmock(log_json=False), @@ -34,7 +34,7 @@ def test_make_list_command_includes_json_but_not_info(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), global_arguments=flexmock(log_json=False), @@ -51,7 +51,7 @@ def test_make_list_command_includes_log_debug(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), global_arguments=flexmock(log_json=False), @@ -68,7 +68,7 @@ def test_make_list_command_includes_json_but_not_debug(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), global_arguments=flexmock(log_json=False), @@ -84,7 +84,7 @@ def test_make_list_command_includes_json(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), global_arguments=flexmock(log_json=False), @@ -100,7 +100,7 @@ def test_make_list_command_includes_log_json(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), global_arguments=flexmock(log_json=True), @@ -118,7 +118,7 @@ def test_make_list_command_includes_lock_wait(): command = module.make_list_command( repository_path='repo', - storage_config={'lock_wait': 5}, + config={'lock_wait': 5}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), global_arguments=flexmock(log_json=False), @@ -136,7 +136,7 @@ def test_make_list_command_includes_archive(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=None, json=False), global_arguments=flexmock(log_json=False), @@ -154,7 +154,7 @@ def test_make_list_command_includes_archive_and_path(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False), global_arguments=flexmock(log_json=False), @@ -170,7 +170,7 @@ def test_make_list_command_includes_local_path(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), global_arguments=flexmock(log_json=False), @@ -193,7 +193,7 @@ def test_make_list_command_includes_remote_path(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), global_arguments=flexmock(log_json=False), @@ -210,7 +210,7 @@ def test_make_list_command_includes_short(): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False, short=True), global_arguments=flexmock(log_json=False), @@ -242,7 +242,7 @@ def test_make_list_command_includes_additional_flags(argument_name): command = module.make_list_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=flexmock( archive=None, @@ -293,7 +293,7 @@ def test_capture_archive_listing_does_not_raise(): module.capture_archive_listing( repository_path='repo', archive='archive', - storage_config=flexmock(), + config=flexmock(), local_borg_version=flexmock(), global_arguments=flexmock(log_json=False), ) @@ -319,7 +319,7 @@ def test_list_archive_calls_borg_with_flags(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=global_arguments, @@ -337,7 +337,7 @@ def test_list_archive_calls_borg_with_flags(): module.list_archive( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=global_arguments, @@ -355,7 +355,7 @@ def test_list_archive_with_archive_and_json_errors(): with pytest.raises(ValueError): module.list_archive( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=flexmock(log_json=False), @@ -382,7 +382,7 @@ def test_list_archive_calls_borg_with_local_path(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=global_arguments, @@ -400,7 +400,7 @@ def test_list_archive_calls_borg_with_local_path(): module.list_archive( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=global_arguments, @@ -450,7 +450,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): module.list_archive( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=flexmock(log_json=False), @@ -477,7 +477,7 @@ def test_list_archive_calls_borg_with_archive(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=global_arguments, @@ -495,7 +495,7 @@ def test_list_archive_calls_borg_with_archive(): module.list_archive( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=global_arguments, @@ -526,7 +526,7 @@ def test_list_archive_without_archive_delegates_to_list_repository(): module.list_archive( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=flexmock(log_json=False), @@ -557,7 +557,7 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos module.list_archive( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=list_arguments, global_arguments=flexmock(log_json=False), @@ -595,7 +595,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( ).and_return(False) flexmock(module).should_receive('make_list_command').with_args( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( archive='archive', paths=None, json=False, find_paths=None, **default_filter_flags @@ -615,7 +615,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( module.list_archive( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( archive='archive', paths=None, json=False, find_paths=None, **altered_filter_flags @@ -654,7 +654,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module.rlist).should_receive('make_rlist_command').with_args( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=argparse.Namespace( repository='repo', short=True, format=None, json=None, **altered_filter_flags @@ -671,7 +671,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module).should_receive('make_list_command').with_args( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( repository='repo', @@ -690,7 +690,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module).should_receive('make_list_command').with_args( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( repository='repo', @@ -724,7 +724,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes module.list_archive( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', list_arguments=argparse.Namespace( repository='repo', diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index bd93cf3d..9ee37f91 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -26,7 +26,7 @@ def test_mount_archive_calls_borg_with_required_flags(): repository_path='repo', archive=None, mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -49,7 +49,7 @@ def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_a repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -67,7 +67,7 @@ def test_mount_archive_without_archive_calls_borg_with_repository_flags_only(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -87,7 +87,7 @@ def test_mount_archive_calls_borg_with_path_flags(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -107,7 +107,7 @@ def test_mount_archive_calls_borg_with_remote_path_flags(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), remote_path='borg1', @@ -126,7 +126,7 @@ def test_mount_archive_calls_borg_with_umask_flags(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={'umask': '0770'}, + config={'umask': '0770'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -144,7 +144,7 @@ def test_mount_archive_calls_borg_with_log_json_flags(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), ) @@ -162,7 +162,7 @@ def test_mount_archive_calls_borg_with_lock_wait_flags(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={'lock_wait': '5'}, + config={'lock_wait': '5'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -181,7 +181,7 @@ def test_mount_archive_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -200,7 +200,7 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_flags(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -224,7 +224,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -244,7 +244,7 @@ def test_mount_archive_calls_borg_with_options_flags(): repository_path='repo', archive='archive', mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -305,7 +305,7 @@ def test_mount_archive_with_date_based_matching_calls_borg_with_date_based_flags repository_path='repo', archive=None, mount_arguments=mount_arguments, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 584d0d50..971eb1e6 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -1,5 +1,4 @@ import logging -from collections import OrderedDict from flexmock import flexmock @@ -22,23 +21,28 @@ BASE_PRUNE_FLAGS = ('--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', def test_make_prune_flags_returns_flags_from_config(): - retention_config = OrderedDict((('keep_daily', 1), ('keep_weekly', 2), ('keep_monthly', 3))) + config = { + 'keep_daily': 1, + 'keep_weekly': 2, + 'keep_monthly': 3, + } flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(config, local_borg_version='1.2.3') assert result == BASE_PRUNE_FLAGS def test_make_prune_flags_accepts_prefix_with_placeholders(): - retention_config = OrderedDict( - (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 - ) + config = { + 'keep_daily': 1, + 'prefix': 'Documents_{hostname}-{now}', # noqa: FS003 + } flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(config, local_borg_version='1.2.3') expected = ( '--keep-daily', @@ -51,13 +55,14 @@ def test_make_prune_flags_accepts_prefix_with_placeholders(): def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives(): - retention_config = OrderedDict( - (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 - ) + config = { + 'keep_daily': 1, + 'prefix': 'Documents_{hostname}-{now}', # noqa: FS003 + } flexmock(module.feature).should_receive('available').and_return(False) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(config, local_borg_version='1.2.3') expected = ( '--keep-daily', @@ -70,12 +75,15 @@ def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives() def test_make_prune_flags_prefers_prefix_to_archive_name_format(): - storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 - retention_config = OrderedDict((('keep_daily', 1), ('prefix', 'bar-'))) + config = { + 'archive_name_format': 'bar-{now}', # noqa: FS003 + 'keep_daily': 1, + 'prefix': 'bar-', + } flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').never() - result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(config, local_borg_version='1.2.3') expected = ( '--keep-daily', @@ -88,14 +96,17 @@ def test_make_prune_flags_prefers_prefix_to_archive_name_format(): def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): - storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 - retention_config = OrderedDict((('keep_daily', 1), ('prefix', None))) + config = { + 'archive_name_format': 'bar-{now}', # noqa: FS003 + 'keep_daily': 1, + 'prefix': None, + } flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( None, 'bar-{now}', '1.2.3' # noqa: FS003 ).and_return(('--match-archives', 'sh:bar-*')) - result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(config, local_borg_version='1.2.3') expected = ( '--keep-daily', @@ -121,8 +132,7 @@ def test_prune_archives_calls_borg_with_flags(): module.prune_archives( dry_run=False, repository_path='repo', - storage_config={}, - retention_config=flexmock(), + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -140,9 +150,8 @@ def test_prune_archives_with_log_info_calls_borg_with_info_flag(): prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( repository_path='repo', - storage_config={}, + config={}, dry_run=False, - retention_config=flexmock(), local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -160,9 +169,8 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_flag(): prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( repository_path='repo', - storage_config={}, + config={}, dry_run=False, - retention_config=flexmock(), local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -179,9 +187,8 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_flag(): prune_arguments = flexmock(stats=False, list_archives=False) module.prune_archives( repository_path='repo', - storage_config={}, + config={}, dry_run=True, - retention_config=flexmock(), local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -199,8 +206,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): module.prune_archives( dry_run=False, repository_path='repo', - storage_config={}, - retention_config=flexmock(), + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), local_path='borg1', @@ -219,8 +225,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): module.prune_archives( dry_run=False, repository_path='repo', - storage_config={}, - retention_config=flexmock(), + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), remote_path='borg1', @@ -239,8 +244,7 @@ def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_ module.prune_archives( dry_run=False, repository_path='repo', - storage_config={}, - retention_config=flexmock(), + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -258,8 +262,7 @@ def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_l module.prune_archives( dry_run=False, repository_path='repo', - storage_config={}, - retention_config=flexmock(), + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -269,7 +272,7 @@ def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_l def test_prune_archives_with_umask_calls_borg_with_umask_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - storage_config = {'umask': '077'} + config = {'umask': '077'} flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077', 'repo'), logging.INFO) @@ -278,8 +281,7 @@ def test_prune_archives_with_umask_calls_borg_with_umask_flags(): module.prune_archives( dry_run=False, repository_path='repo', - storage_config=storage_config, - retention_config=flexmock(), + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -297,8 +299,7 @@ def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): module.prune_archives( dry_run=False, repository_path='repo', - storage_config={}, - retention_config=flexmock(), + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), prune_arguments=prune_arguments, @@ -308,7 +309,7 @@ def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - storage_config = {'lock_wait': 5} + config = {'lock_wait': 5} flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) @@ -317,8 +318,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): module.prune_archives( dry_run=False, repository_path='repo', - storage_config=storage_config, - retention_config=flexmock(), + config=config, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -336,8 +336,7 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): module.prune_archives( dry_run=False, repository_path='repo', - storage_config={'extra_borg_options': {'prune': '--extra --options'}}, - retention_config=flexmock(), + config={'extra_borg_options': {'prune': '--extra --options'}}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -400,8 +399,7 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag module.prune_archives( dry_run=False, repository_path='repo', - storage_config={}, - retention_config=flexmock(), + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index 2f71a8ff..be11a829 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -46,7 +46,7 @@ def test_create_repository_calls_borg_with_flags(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -67,7 +67,7 @@ def test_create_repository_with_dry_run_skips_borg_call(): module.create_repository( dry_run=True, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -92,7 +92,7 @@ def test_create_repository_raises_for_borg_rcreate_error(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -112,7 +112,7 @@ def test_create_repository_skips_creation_when_repository_already_exists(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -128,7 +128,7 @@ def test_create_repository_raises_for_unknown_rinfo_command_error(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -149,7 +149,7 @@ def test_create_repository_with_source_repository_calls_borg_with_other_repo_fla module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -171,7 +171,7 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -193,7 +193,7 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -215,7 +215,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -237,7 +237,7 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -260,7 +260,7 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -282,7 +282,7 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -303,7 +303,7 @@ def test_create_repository_with_log_json_calls_borg_with_log_json_flag(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=True), encryption_mode='repokey', @@ -324,7 +324,7 @@ def test_create_repository_with_lock_wait_calls_borg_with_lock_wait_flag(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={'lock_wait': 5}, + config={'lock_wait': 5}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -345,7 +345,7 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -367,7 +367,7 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): module.create_repository( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -389,7 +389,7 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options module.create_repository( dry_run=False, repository_path='repo', - storage_config={'extra_borg_options': {'rcreate': '--extra --options'}}, + config={'extra_borg_options': {'rcreate': '--extra --options'}}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index a6e3f08c..8628b9aa 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -27,7 +27,7 @@ def test_display_repository_info_calls_borg_with_flags(): module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), global_arguments=flexmock(log_json=False), @@ -49,7 +49,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), global_arguments=flexmock(log_json=False), @@ -76,7 +76,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag(): insert_logging_mock(logging.INFO) module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), global_arguments=flexmock(log_json=False), @@ -102,7 +102,7 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out insert_logging_mock(logging.INFO) json_output = module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), global_arguments=flexmock(log_json=False), @@ -132,7 +132,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag(): module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), global_arguments=flexmock(log_json=False), @@ -158,7 +158,7 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou insert_logging_mock(logging.DEBUG) json_output = module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), global_arguments=flexmock(log_json=False), @@ -185,7 +185,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_flag(): json_output = module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), global_arguments=flexmock(log_json=False), @@ -214,7 +214,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), global_arguments=flexmock(log_json=False), @@ -242,7 +242,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), global_arguments=flexmock(log_json=False), @@ -270,7 +270,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags(): module.display_repository_info( repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), global_arguments=flexmock(log_json=True), @@ -280,7 +280,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags(): def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - storage_config = {'lock_wait': 5} + config = {'lock_wait': 5} flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_flags').and_return( ( @@ -298,7 +298,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags( module.display_repository_info( repository_path='repo', - storage_config=storage_config, + config=config, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), global_arguments=flexmock(log_json=False), diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 76bda987..65fcef1d 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -23,7 +23,7 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name(): module.resolve_archive_name( 'repo', archive, - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -43,7 +43,7 @@ def test_resolve_archive_name_calls_borg_with_flags(): module.resolve_archive_name( 'repo', 'latest', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -64,7 +64,7 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): module.resolve_archive_name( 'repo', 'latest', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -85,7 +85,7 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): module.resolve_archive_name( 'repo', 'latest', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -105,7 +105,7 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): module.resolve_archive_name( 'repo', 'latest', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), local_path='borg1', @@ -126,7 +126,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags module.resolve_archive_name( 'repo', 'latest', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), remote_path='borg1', @@ -146,7 +146,7 @@ def test_resolve_archive_name_without_archives_raises(): module.resolve_archive_name( 'repo', 'latest', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -165,7 +165,7 @@ def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): module.resolve_archive_name( 'repo', 'latest', - storage_config={}, + config={}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), ) @@ -186,7 +186,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): module.resolve_archive_name( 'repo', 'latest', - storage_config={'lock_wait': 'okay'}, + config={'lock_wait': 'okay'}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), ) @@ -205,7 +205,7 @@ def test_make_rlist_command_includes_log_info(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None @@ -227,7 +227,7 @@ def test_make_rlist_command_includes_json_but_not_info(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None @@ -249,7 +249,7 @@ def test_make_rlist_command_includes_log_debug(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None @@ -271,7 +271,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None @@ -292,7 +292,7 @@ def test_make_rlist_command_includes_json(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None @@ -315,7 +315,7 @@ def test_make_rlist_command_includes_log_json(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None @@ -338,7 +338,7 @@ def test_make_rlist_command_includes_lock_wait(): command = module.make_rlist_command( repository_path='repo', - storage_config={'lock_wait': 5}, + config={'lock_wait': 5}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None @@ -359,7 +359,7 @@ def test_make_rlist_command_includes_local_path(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None @@ -383,7 +383,7 @@ def test_make_rlist_command_includes_remote_path(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None @@ -407,7 +407,7 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), global_arguments=flexmock(log_json=False), @@ -426,7 +426,7 @@ def test_make_rlist_command_prefers_prefix_over_archive_name_format(): command = module.make_rlist_command( repository_path='repo', - storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), global_arguments=flexmock(log_json=False), @@ -445,7 +445,7 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives() command = module.make_rlist_command( repository_path='repo', - storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None @@ -466,7 +466,7 @@ def test_make_rlist_command_includes_short(): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None, short=True @@ -501,7 +501,7 @@ def test_make_rlist_command_includes_additional_flags(argument_name): command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, @@ -534,7 +534,7 @@ def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_f command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, @@ -560,7 +560,7 @@ def test_list_repository_calls_borg_with_flags(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, global_arguments=global_arguments, @@ -577,7 +577,7 @@ def test_list_repository_calls_borg_with_flags(): module.list_repository( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, global_arguments=global_arguments, @@ -594,7 +594,7 @@ def test_list_repository_with_json_returns_borg_output(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, global_arguments=global_arguments, @@ -607,7 +607,7 @@ def test_list_repository_with_json_returns_borg_output(): assert ( module.list_repository( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, global_arguments=global_arguments, @@ -628,7 +628,7 @@ def test_make_rlist_command_with_date_based_matching_calls_borg_with_date_based_ command = module.make_rlist_command( repository_path='repo', - storage_config={}, + config={}, local_borg_version='1.2.3', rlist_arguments=flexmock( archive=None, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 6cd9530b..3e1aa804 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -27,7 +27,7 @@ def test_transfer_archives_calls_borg_with_flags(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None @@ -58,7 +58,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): module.transfer_archives( dry_run=True, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None @@ -86,7 +86,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None @@ -115,7 +115,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None @@ -145,7 +145,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive='archive', progress=None, match_archives=None, source_repository=None @@ -175,7 +175,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None @@ -205,7 +205,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None @@ -233,7 +233,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None @@ -265,7 +265,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None @@ -297,7 +297,7 @@ def test_transfer_archives_with_log_json_calls_borg_with_log_json_flags(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None @@ -316,7 +316,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) - storage_config = {'lock_wait': 5} + config = {'lock_wait': 5} flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'transfer', '--lock-wait', '5', '--repo', 'repo'), @@ -329,7 +329,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config=storage_config, + config=config, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None @@ -357,7 +357,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=True, match_archives=None, source_repository=None @@ -389,7 +389,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, @@ -423,7 +423,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository='other' @@ -465,7 +465,7 @@ def test_transfer_archives_with_date_based_matching_calls_borg_with_date_based_f module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + config={}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), transfer_arguments=flexmock( diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 89eda4ce..e94bdd8c 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -15,7 +15,7 @@ def test_run_configuration_runs_actions_for_each_repository(): flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return( expected_results[1:] ) - config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}} + config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} arguments = {'global': flexmock(monitoring_verbosity=1)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -29,7 +29,7 @@ def test_run_configuration_with_invalid_borg_version_errors(): flexmock(module.command).should_receive('execute_hook').never() flexmock(module.dispatch).should_receive('call_hooks').never() flexmock(module).should_receive('run_actions').never() - config = {'location': {'repositories': ['foo']}} + config = {'repositories': ['foo']} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock()} list(module.run_configuration('test.yaml', config, arguments)) @@ -44,7 +44,7 @@ def test_run_configuration_logs_monitor_start_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').never() - config = {'location': {'repositories': ['foo']}} + config = {'repositories': ['foo']} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -59,7 +59,7 @@ def test_run_configuration_bails_for_monitor_start_soft_failure(): flexmock(module.dispatch).should_receive('call_hooks').and_raise(error) flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').never() - config = {'location': {'repositories': ['foo']}} + config = {'repositories': ['foo']} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -75,7 +75,7 @@ def test_run_configuration_logs_actions_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': [{'path': 'foo'}]}} + config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -91,7 +91,7 @@ def test_run_configuration_bails_for_actions_soft_failure(): flexmock(module).should_receive('run_actions').and_raise(error) flexmock(module).should_receive('log_error_records').never() flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': [{'path': 'foo'}]}} + config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -108,7 +108,7 @@ def test_run_configuration_logs_monitor_log_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) - config = {'location': {'repositories': [{'path': 'foo'}]}} + config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -126,7 +126,7 @@ def test_run_configuration_bails_for_monitor_log_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': [{'path': 'foo'}]}} + config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -143,7 +143,7 @@ def test_run_configuration_logs_monitor_finish_error(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) - config = {'location': {'repositories': [{'path': 'foo'}]}} + config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -161,7 +161,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'location': {'repositories': [{'path': 'foo'}]}} + config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -176,7 +176,7 @@ def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_ar flexmock(module.dispatch).should_receive('call_hooks').never() flexmock(module).should_receive('run_actions').and_return([]) - config = {'location': {'repositories': [{'path': 'foo'}]}} + config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=-2, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == [] @@ -191,7 +191,7 @@ def test_run_configuration_logs_on_error_hook_error(): expected_results[:1] ).and_return(expected_results[1:]) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': [{'path': 'foo'}]}} + config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -207,7 +207,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) - config = {'location': {'repositories': [{'path': 'foo'}]}} + config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -222,7 +222,7 @@ def test_run_configuration_retries_soft_error(): flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once() - config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} + config = {'repositories': [{'path': 'foo'}], 'retries': 1} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == [] @@ -245,7 +245,7 @@ def test_run_configuration_retries_hard_error(): 'foo: Error running actions for repository', OSError, ).and_return(error_logs) - config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}} + config = {'repositories': [{'path': 'foo'}], 'retries': 1} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -263,7 +263,7 @@ def test_run_configuration_repos_ordered(): flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError ).and_return(expected_results[1:]).ordered() - config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}} + config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results @@ -295,8 +295,8 @@ def test_run_configuration_retries_round_robin(): 'bar: Error running actions for repository', OSError ).and_return(bar_error_logs).ordered() config = { - 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, - 'storage': {'retries': 1}, + 'repositories': [{'path': 'foo'}, {'path': 'bar'}], + 'retries': 1, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -327,8 +327,8 @@ def test_run_configuration_retries_one_passes(): 'bar: Error running actions for repository', OSError ).and_return(error_logs).ordered() config = { - 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, - 'storage': {'retries': 1}, + 'repositories': [{'path': 'foo'}, {'path': 'bar'}], + 'retries': 1, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -369,8 +369,9 @@ def test_run_configuration_retry_wait(): 'foo: Error running actions for repository', OSError ).and_return(error_logs).ordered() config = { - 'location': {'repositories': [{'path': 'foo'}]}, - 'storage': {'retries': 3, 'retry_wait': 10}, + 'repositories': [{'path': 'foo'}], + 'retries': 3, + 'retry_wait': 10, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -407,8 +408,9 @@ def test_run_configuration_retries_timeout_multiple_repos(): 'bar: Error running actions for repository', OSError ).and_return(error_logs).ordered() config = { - 'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}, - 'storage': {'retries': 1, 'retry_wait': 10}, + 'repositories': [{'path': 'foo'}, {'path': 'bar'}], + 'retries': 1, + 'retry_wait': 10, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -424,11 +426,7 @@ def test_run_actions_runs_rcreate(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -444,9 +442,7 @@ def test_run_actions_adds_log_file_to_hook_context(): flexmock(borgmatic.actions.create).should_receive('run_create').with_args( config_filename=object, repository={'path': 'repo'}, - location={'repositories': []}, - storage=object, - hooks={}, + config={'repositories': []}, hook_context={'repository': 'repo', 'repositories': '', 'log_file': 'foo'}, local_borg_version=object, create_arguments=object, @@ -460,11 +456,7 @@ def test_run_actions_adds_log_file_to_hook_context(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -483,11 +475,7 @@ def test_run_actions_runs_transfer(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'transfer': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -506,11 +494,7 @@ def test_run_actions_runs_create(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -529,11 +513,7 @@ def test_run_actions_runs_prune(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'prune': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -551,11 +531,7 @@ def test_run_actions_runs_compact(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'compact': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -574,11 +550,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -597,11 +569,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -619,11 +587,7 @@ def test_run_actions_runs_extract(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'extract': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -641,11 +605,7 @@ def test_run_actions_runs_export_tar(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'export-tar': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -663,11 +623,7 @@ def test_run_actions_runs_mount(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'mount': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -685,11 +641,7 @@ def test_run_actions_runs_restore(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'restore': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -708,11 +660,7 @@ def test_run_actions_runs_rlist(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rlist': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -732,11 +680,7 @@ def test_run_actions_runs_list(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'list': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -756,11 +700,7 @@ def test_run_actions_runs_rinfo(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rinfo': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -780,11 +720,7 @@ def test_run_actions_runs_info(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'info': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -803,11 +739,7 @@ def test_run_actions_runs_break_lock(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'break-lock': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -825,11 +757,7 @@ def test_run_actions_runs_borg(): module.run_actions( arguments={'global': flexmock(dry_run=False, log_file='foo'), 'borg': flexmock()}, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -852,11 +780,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): 'restore': flexmock(), }, config_filename=flexmock(), - location={'repositories': []}, - storage=flexmock(), - retention=flexmock(), - consistency=flexmock(), - hooks={}, + config={'repositories': []}, local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -951,15 +875,11 @@ def test_log_error_records_generates_nothing_for_other_error(): def test_get_local_path_uses_configuration_value(): - assert module.get_local_path({'test.yaml': {'location': {'local_path': 'borg1'}}}) == 'borg1' - - -def test_get_local_path_without_location_defaults_to_borg(): - assert module.get_local_path({'test.yaml': {}}) == 'borg' + assert module.get_local_path({'test.yaml': {'local_path': 'borg1'}}) == 'borg1' def test_get_local_path_without_local_path_defaults_to_borg(): - assert module.get_local_path({'test.yaml': {'location': {}}}) == 'borg' + assert module.get_local_path({'test.yaml': {}}) == 'borg' def test_collect_highlander_action_summary_logs_info_for_success_with_bootstrap(): diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 7855592c..1dcbf074 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -13,35 +13,20 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): 'type': 'object', 'properties': OrderedDict( [ - ( - 'section1', - { - 'type': 'object', - 'properties': {'field1': OrderedDict([('example', 'Example 1')])}, - }, - ), - ( - 'section2', - { - 'type': 'object', - 'properties': OrderedDict( - [ - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - ), + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), ] ), } - config = module._schema_to_sample_configuration(schema) + config = module.schema_to_sample_configuration(schema) assert config == OrderedDict( [ - ('section1', OrderedDict([('field1', 'Example 1')])), - ('section2', OrderedDict([('field2', 'Example 2'), ('field3', 'Example 3')])), + ('field1', 'Example 1'), + ('field2', 'Example 2'), + ('field3', 'Example 3'), ] ) @@ -51,7 +36,7 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_strings_wit flexmock(module).should_receive('add_comments_to_configuration_sequence') schema = {'type': 'array', 'items': {'type': 'string'}, 'example': ['hi']} - config = module._schema_to_sample_configuration(schema) + config = module.schema_to_sample_configuration(schema) assert config == ['hi'] @@ -70,7 +55,7 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e }, } - config = module._schema_to_sample_configuration(schema) + config = module.schema_to_sample_configuration(schema) assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])] @@ -79,7 +64,7 @@ def test_schema_to_sample_configuration_with_unsupported_schema_raises(): schema = {'gobbledygook': [{'type': 'not-your'}]} with pytest.raises(ValueError): - module._schema_to_sample_configuration(schema) + module.schema_to_sample_configuration(schema) def test_merge_source_configuration_into_destination_inserts_map_fields(): diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 6393d902..633e0781 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -1,4 +1,5 @@ import pytest +from flexmock import flexmock from borgmatic.config import normalize as module @@ -7,138 +8,220 @@ from borgmatic.config import normalize as module 'config,expected_config,produces_logs', ( ( - {'location': {'exclude_if_present': '.nobackup'}}, - {'location': {'exclude_if_present': ['.nobackup']}}, + {'location': {'foo': 'bar', 'baz': 'quux'}}, + {'foo': 'bar', 'baz': 'quux'}, True, ), ( - {'location': {'exclude_if_present': ['.nobackup']}}, - {'location': {'exclude_if_present': ['.nobackup']}}, + {'retention': {'foo': 'bar', 'baz': 'quux'}}, + {'foo': 'bar', 'baz': 'quux'}, + True, + ), + ( + {'consistency': {'foo': 'bar', 'baz': 'quux'}}, + {'foo': 'bar', 'baz': 'quux'}, + True, + ), + ( + {'output': {'foo': 'bar', 'baz': 'quux'}}, + {'foo': 'bar', 'baz': 'quux'}, + True, + ), + ( + {'hooks': {'foo': 'bar', 'baz': 'quux'}}, + {'foo': 'bar', 'baz': 'quux'}, + True, + ), + ( + {'location': {'foo': 'bar'}, 'storage': {'baz': 'quux'}}, + {'foo': 'bar', 'baz': 'quux'}, + True, + ), + ( + {'foo': 'bar', 'baz': 'quux'}, + {'foo': 'bar', 'baz': 'quux'}, False, ), ( - {'location': {'source_directories': ['foo', 'bar']}}, - {'location': {'source_directories': ['foo', 'bar']}}, + {'location': {'prefix': 'foo'}, 'consistency': {'prefix': 'foo'}}, + {'prefix': 'foo'}, + True, + ), + ( + {'location': {'prefix': 'foo'}, 'consistency': {'prefix': 'foo'}}, + {'prefix': 'foo'}, + True, + ), + ( + {'location': {'prefix': 'foo'}, 'consistency': {'bar': 'baz'}}, + {'prefix': 'foo', 'bar': 'baz'}, + True, + ), + ( + {'storage': {'umask': 'foo'}, 'hooks': {'umask': 'foo'}}, + {'umask': 'foo'}, + True, + ), + ( + {'storage': {'umask': 'foo'}, 'hooks': {'umask': 'foo'}}, + {'umask': 'foo'}, + True, + ), + ( + {'storage': {'umask': 'foo'}, 'hooks': {'bar': 'baz'}}, + {'umask': 'foo', 'bar': 'baz'}, + True, + ), + ( + {'location': {'bar': 'baz'}, 'consistency': {'prefix': 'foo'}}, + {'bar': 'baz', 'prefix': 'foo'}, + True, + ), + ( + {}, + {}, + False, + ), + ), +) +def test_normalize_sections_moves_section_options_to_global_scope( + config, expected_config, produces_logs +): + logs = module.normalize_sections('test.yaml', config) + + assert config == expected_config + + if produces_logs: + assert logs + else: + assert logs == [] + + +def test_normalize_sections_with_different_prefix_values_raises(): + config = {'location': {'prefix': 'foo'}, 'consistency': {'prefix': 'bar'}} + + with pytest.raises(ValueError): + module.normalize_sections('test.yaml', config) + + +def test_normalize_sections_with_different_umask_values_raises(): + config = {'storage': {'umask': 'foo'}, 'hooks': {'umask': 'bar'}} + + with pytest.raises(ValueError): + module.normalize_sections('test.yaml', config) + + +@pytest.mark.parametrize( + 'config,expected_config,produces_logs', + ( + ( + {'exclude_if_present': '.nobackup'}, + {'exclude_if_present': ['.nobackup']}, + True, + ), + ( + {'exclude_if_present': ['.nobackup']}, + {'exclude_if_present': ['.nobackup']}, False, ), ( - {'location': None}, - {'location': None}, + {'source_directories': ['foo', 'bar']}, + {'source_directories': ['foo', 'bar']}, False, ), ( - {'storage': {'compression': 'yes_please'}}, - {'storage': {'compression': 'yes_please'}}, + {'compression': 'yes_please'}, + {'compression': 'yes_please'}, False, ), ( - {'storage': None}, - {'storage': None}, + {'healthchecks': 'https://example.com'}, + {'healthchecks': {'ping_url': 'https://example.com'}}, + True, + ), + ( + {'cronitor': 'https://example.com'}, + {'cronitor': {'ping_url': 'https://example.com'}}, + True, + ), + ( + {'pagerduty': 'https://example.com'}, + {'pagerduty': {'integration_key': 'https://example.com'}}, + True, + ), + ( + {'cronhub': 'https://example.com'}, + {'cronhub': {'ping_url': 'https://example.com'}}, + True, + ), + ( + {'checks': ['archives']}, + {'checks': [{'name': 'archives'}]}, + True, + ), + ( + {'checks': ['archives']}, + {'checks': [{'name': 'archives'}]}, + True, + ), + ( + {'numeric_owner': False}, + {'numeric_ids': False}, + True, + ), + ( + {'bsd_flags': False}, + {'flags': False}, + True, + ), + ( + {'remote_rate_limit': False}, + {'upload_rate_limit': False}, + True, + ), + ( + {'repositories': ['foo@bar:/repo']}, + {'repositories': [{'path': 'ssh://foo@bar/repo'}]}, + True, + ), + ( + {'repositories': ['foo@bar:repo']}, + {'repositories': [{'path': 'ssh://foo@bar/./repo'}]}, + True, + ), + ( + {'repositories': ['foo@bar:~/repo']}, + {'repositories': [{'path': 'ssh://foo@bar/~/repo'}]}, + True, + ), + ( + {'repositories': ['ssh://foo@bar:1234/repo']}, + {'repositories': [{'path': 'ssh://foo@bar:1234/repo'}]}, + True, + ), + ( + {'repositories': ['file:///repo']}, + {'repositories': [{'path': '/repo'}]}, + True, + ), + ( + {'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}]}, + {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': 'foo'}]}, + True, + ), + ( + {'repositories': [{'path': 'file:///repo', 'label': 'foo'}]}, + {'repositories': [{'path': '/repo', 'label': 'foo'}]}, False, ), ( - {'hooks': {'healthchecks': 'https://example.com'}}, - {'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}}, - True, - ), - ( - {'hooks': {'cronitor': 'https://example.com'}}, - {'hooks': {'cronitor': {'ping_url': 'https://example.com'}}}, - True, - ), - ( - {'hooks': {'pagerduty': 'https://example.com'}}, - {'hooks': {'pagerduty': {'integration_key': 'https://example.com'}}}, - True, - ), - ( - {'hooks': {'cronhub': 'https://example.com'}}, - {'hooks': {'cronhub': {'ping_url': 'https://example.com'}}}, - True, - ), - ( - {'hooks': None}, - {'hooks': None}, + {'repositories': [{'path': '/repo', 'label': 'foo'}]}, + {'repositories': [{'path': '/repo', 'label': 'foo'}]}, False, ), ( - {'consistency': {'checks': ['archives']}}, - {'consistency': {'checks': [{'name': 'archives'}]}}, - True, - ), - ( - {'consistency': {'checks': ['archives']}}, - {'consistency': {'checks': [{'name': 'archives'}]}}, - True, - ), - ( - {'consistency': None}, - {'consistency': None}, - False, - ), - ( - {'location': {'numeric_owner': False}}, - {'location': {'numeric_ids': False}}, - True, - ), - ( - {'location': {'bsd_flags': False}}, - {'location': {'flags': False}}, - True, - ), - ( - {'storage': {'remote_rate_limit': False}}, - {'storage': {'upload_rate_limit': False}}, - True, - ), - ( - {'location': {'repositories': ['foo@bar:/repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/repo'}]}}, - True, - ), - ( - {'location': {'repositories': ['foo@bar:repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/./repo'}]}}, - True, - ), - ( - {'location': {'repositories': ['foo@bar:~/repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo'}]}}, - True, - ), - ( - {'location': {'repositories': ['ssh://foo@bar:1234/repo']}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo'}]}}, - True, - ), - ( - {'location': {'repositories': ['file:///repo']}}, - {'location': {'repositories': [{'path': '/repo'}]}}, - True, - ), - ( - {'location': {'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}]}}, - {'location': {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': 'foo'}]}}, - True, - ), - ( - {'location': {'repositories': [{'path': 'file:///repo', 'label': 'foo'}]}}, - {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, - False, - ), - ( - {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, - {'location': {'repositories': [{'path': '/repo', 'label': 'foo'}]}}, - False, - ), - ( - {'consistency': {'prefix': 'foo'}}, - {'consistency': {'prefix': 'foo'}}, - True, - ), - ( - {'retention': {'prefix': 'foo'}}, - {'retention': {'prefix': 'foo'}}, + {'prefix': 'foo'}, + {'prefix': 'foo'}, True, ), ), @@ -146,6 +229,8 @@ from borgmatic.config import normalize as module def test_normalize_applies_hard_coded_normalization_to_config( config, expected_config, produces_logs ): + flexmock(module).should_receive('normalize_sections').and_return([]) + logs = module.normalize('test.yaml', config) assert config == expected_config @@ -157,12 +242,12 @@ def test_normalize_applies_hard_coded_normalization_to_config( def test_normalize_raises_error_if_repository_data_is_not_consistent(): + flexmock(module).should_receive('normalize_sections').and_return([]) + with pytest.raises(TypeError): module.normalize( 'test.yaml', { - 'location': { - 'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}, 'file:///repo'] - } + 'repositories': [{'path': 'foo@bar:/repo', 'label': 'foo'}, 'file:///repo'], }, ) diff --git a/tests/unit/config/test_override.py b/tests/unit/config/test_override.py index a5d62eb4..38b07e40 100644 --- a/tests/unit/config/test_override.py +++ b/tests/unit/config/test_override.py @@ -32,54 +32,76 @@ def test_set_values_with_one_key_overwrites_existing_key(): def test_set_values_with_multiple_keys_creates_hierarchy(): config = {} - module.set_values(config, ('section', 'key'), 'value') + module.set_values(config, ('option', 'suboption'), 'value') - assert config == {'section': {'key': 'value'}} + assert config == {'option': {'suboption': 'value'}} def test_set_values_with_multiple_keys_updates_hierarchy(): - config = {'section': {'other': 'other_value'}} - module.set_values(config, ('section', 'key'), 'value') + config = {'option': {'other': 'other_value'}} + module.set_values(config, ('option', 'key'), 'value') - assert config == {'section': {'key': 'value', 'other': 'other_value'}} + assert config == {'option': {'key': 'value', 'other': 'other_value'}} + + +@pytest.mark.parametrize( + 'key,expected_key', + ( + (('foo', 'bar'), ('foo', 'bar')), + (('location', 'foo'), ('foo',)), + (('storage', 'foo'), ('foo',)), + (('retention', 'foo'), ('foo',)), + (('consistency', 'foo'), ('foo',)), + (('output', 'foo'), ('foo',)), + (('hooks', 'foo', 'bar'), ('foo', 'bar')), + (('foo', 'hooks'), ('foo', 'hooks')), + ), +) +def test_strip_section_names_passes_through_key_without_section_name(key, expected_key): + assert module.strip_section_names(key) == expected_key def test_parse_overrides_splits_keys_and_values(): + flexmock(module).should_receive('strip_section_names').replace_with(lambda value: value) flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value) - raw_overrides = ['section.my_option=value1', 'section.other_option=value2'] + raw_overrides = ['option.my_option=value1', 'other_option=value2'] expected_result = ( - (('section', 'my_option'), 'value1'), - (('section', 'other_option'), 'value2'), + (('option', 'my_option'), 'value1'), + (('other_option'), 'value2'), ) module.parse_overrides(raw_overrides) == expected_result def test_parse_overrides_allows_value_with_equal_sign(): + flexmock(module).should_receive('strip_section_names').replace_with(lambda value: value) flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value) - raw_overrides = ['section.option=this===value'] - expected_result = ((('section', 'option'), 'this===value'),) + raw_overrides = ['option=this===value'] + expected_result = ((('option',), 'this===value'),) module.parse_overrides(raw_overrides) == expected_result def test_parse_overrides_raises_on_missing_equal_sign(): + flexmock(module).should_receive('strip_section_names').replace_with(lambda value: value) flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value) - raw_overrides = ['section.option'] + raw_overrides = ['option'] with pytest.raises(ValueError): module.parse_overrides(raw_overrides) def test_parse_overrides_raises_on_invalid_override_value(): + flexmock(module).should_receive('strip_section_names').replace_with(lambda value: value) flexmock(module).should_receive('convert_value_type').and_raise(ruamel.yaml.parser.ParserError) - raw_overrides = ['section.option=[in valid]'] + raw_overrides = ['option=[in valid]'] with pytest.raises(ValueError): module.parse_overrides(raw_overrides) def test_parse_overrides_allows_value_with_single_key(): + flexmock(module).should_receive('strip_section_names').replace_with(lambda value: value) flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value) raw_overrides = ['option=value'] expected_result = ((('option',), 'value'),) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 4ab2b762..182cbd17 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -68,9 +68,9 @@ def test_apply_logical_validation_raises_if_unknown_repository_in_check_reposito module.apply_logical_validation( 'config.yaml', { - 'location': {'repositories': ['repo.borg', 'other.borg']}, - 'retention': {'keep_secondly': 1000}, - 'consistency': {'check_repositories': ['repo.borg', 'unknown.borg']}, + 'repositories': ['repo.borg', 'other.borg'], + 'keep_secondly': 1000, + 'check_repositories': ['repo.borg', 'unknown.borg'], }, ) @@ -79,9 +79,9 @@ def test_apply_logical_validation_does_not_raise_if_known_repository_path_in_che module.apply_logical_validation( 'config.yaml', { - 'location': {'repositories': [{'path': 'repo.borg'}, {'path': 'other.borg'}]}, - 'retention': {'keep_secondly': 1000}, - 'consistency': {'check_repositories': ['repo.borg']}, + 'repositories': [{'path': 'repo.borg'}, {'path': 'other.borg'}], + 'keep_secondly': 1000, + 'check_repositories': ['repo.borg'], }, ) @@ -90,14 +90,12 @@ def test_apply_logical_validation_does_not_raise_if_known_repository_label_in_ch module.apply_logical_validation( 'config.yaml', { - 'location': { - 'repositories': [ - {'path': 'repo.borg', 'label': 'my_repo'}, - {'path': 'other.borg', 'label': 'other_repo'}, - ] - }, - 'retention': {'keep_secondly': 1000}, - 'consistency': {'check_repositories': ['my_repo']}, + 'repositories': [ + {'path': 'repo.borg', 'label': 'my_repo'}, + {'path': 'other.borg', 'label': 'other_repo'}, + ], + 'keep_secondly': 1000, + 'check_repositories': ['my_repo'], }, ) @@ -106,15 +104,15 @@ def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_pref module.apply_logical_validation( 'config.yaml', { - 'storage': {'archive_name_format': '{hostname}-{now}'}, # noqa: FS003 - 'retention': {'prefix': '{hostname}-'}, # noqa: FS003 - 'consistency': {'prefix': '{hostname}-'}, # noqa: FS003 + 'archive_name_format': '{hostname}-{now}', # noqa: FS003 + 'prefix': '{hostname}-', # noqa: FS003 + 'prefix': '{hostname}-', # noqa: FS003 }, ) def test_apply_logical_validation_does_not_raise_otherwise(): - module.apply_logical_validation('config.yaml', {'retention': {'keep_secondly': 1000}}) + module.apply_logical_validation('config.yaml', {'keep_secondly': 1000}) def test_normalize_repository_path_passes_through_remote_repository(): @@ -157,22 +155,20 @@ def test_guard_configuration_contains_repository_does_not_raise_when_repository_ ) module.guard_configuration_contains_repository( - repository='repo', configurations={'config.yaml': {'location': {'repositories': ['repo']}}} + repository='repo', configurations={'config.yaml': {'repositories': ['repo']}} ) def test_guard_configuration_contains_repository_does_not_raise_when_repository_label_in_config(): module.guard_configuration_contains_repository( repository='repo', - configurations={ - 'config.yaml': {'location': {'repositories': [{'path': 'foo/bar', 'label': 'repo'}]}} - }, + configurations={'config.yaml': {'repositories': [{'path': 'foo/bar', 'label': 'repo'}]}}, ) def test_guard_configuration_contains_repository_does_not_raise_when_repository_not_given(): module.guard_configuration_contains_repository( - repository=None, configurations={'config.yaml': {'location': {'repositories': ['repo']}}} + repository=None, configurations={'config.yaml': {'repositories': ['repo']}} ) @@ -184,7 +180,7 @@ def test_guard_configuration_contains_repository_errors_when_repository_missing_ with pytest.raises(ValueError): module.guard_configuration_contains_repository( repository='nope', - configurations={'config.yaml': {'location': {'repositories': ['repo', 'repo2']}}}, + configurations={'config.yaml': {'repositories': ['repo', 'repo2']}}, ) @@ -197,8 +193,8 @@ def test_guard_configuration_contains_repository_errors_when_repository_matches_ module.guard_configuration_contains_repository( repository='repo', configurations={ - 'config.yaml': {'location': {'repositories': ['repo', 'repo2']}}, - 'other.yaml': {'location': {'repositories': ['repo']}}, + 'config.yaml': {'repositories': ['repo', 'repo2']}, + 'other.yaml': {'repositories': ['repo']}, }, ) @@ -207,26 +203,26 @@ def test_guard_single_repository_selected_raises_when_multiple_repositories_conf with pytest.raises(ValueError): module.guard_single_repository_selected( repository=None, - configurations={'config.yaml': {'location': {'repositories': ['repo', 'repo2']}}}, + configurations={'config.yaml': {'repositories': ['repo', 'repo2']}}, ) def test_guard_single_repository_selected_does_not_raise_when_single_repository_configured_and_none_selected(): module.guard_single_repository_selected( repository=None, - configurations={'config.yaml': {'location': {'repositories': ['repo']}}}, + configurations={'config.yaml': {'repositories': ['repo']}}, ) def test_guard_single_repository_selected_does_not_raise_when_no_repositories_configured_and_one_selected(): module.guard_single_repository_selected( repository='repo', - configurations={'config.yaml': {'location': {'repositories': []}}}, + configurations={'config.yaml': {'repositories': []}}, ) def test_guard_single_repository_selected_does_not_raise_when_repositories_configured_and_one_selected(): module.guard_single_repository_selected( repository='repo', - configurations={'config.yaml': {'location': {'repositories': ['repo', 'repo2']}}}, + configurations={'config.yaml': {'repositories': ['repo', 'repo2']}}, ) diff --git a/tests/unit/hooks/test_cronhub.py b/tests/unit/hooks/test_cronhub.py index 2941592b..edb167ad 100644 --- a/tests/unit/hooks/test_cronhub.py +++ b/tests/unit/hooks/test_cronhub.py @@ -11,6 +11,7 @@ def test_ping_monitor_rewrites_ping_url_for_start_state(): module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, @@ -26,6 +27,7 @@ def test_ping_monitor_rewrites_ping_url_and_state_for_start_state(): module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, @@ -41,6 +43,7 @@ def test_ping_monitor_rewrites_ping_url_for_finish_state(): module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, @@ -55,7 +58,12 @@ def test_ping_monitor_rewrites_ping_url_for_fail_state(): ).and_return(flexmock(ok=True)) module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + {}, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, ) @@ -64,7 +72,12 @@ def test_ping_monitor_dry_run_does_not_hit_ping_url(): flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True + hook_config, + {}, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=True, ) @@ -77,6 +90,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.ping_monitor( hook_config, + (), 'config.yaml', module.monitor.State.START, monitoring_log_level=1, @@ -97,6 +111,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, @@ -104,11 +119,13 @@ def test_ping_monitor_with_other_error_logs_warning(): ) -def test_ping_monitor_with_unsupported_monitoring_state(): +def test_ping_monitor_with_unsupported_monitoring_state_bails(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() + module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py index 12b96855..9daa2c11 100644 --- a/tests/unit/hooks/test_cronitor.py +++ b/tests/unit/hooks/test_cronitor.py @@ -11,6 +11,7 @@ def test_ping_monitor_hits_ping_url_for_start_state(): module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, @@ -26,6 +27,7 @@ def test_ping_monitor_hits_ping_url_for_finish_state(): module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, @@ -40,7 +42,12 @@ def test_ping_monitor_hits_ping_url_for_fail_state(): ).and_return(flexmock(ok=True)) module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + {}, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, ) @@ -49,7 +56,12 @@ def test_ping_monitor_dry_run_does_not_hit_ping_url(): flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True + hook_config, + {}, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=True, ) @@ -62,6 +74,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, @@ -82,6 +95,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, @@ -89,11 +103,13 @@ def test_ping_monitor_with_other_error_logs_warning(): ) -def test_ping_monitor_with_unsupported_monitoring_state(): +def test_ping_monitor_with_unsupported_monitoring_state_bails(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() + module.ping_monitor( hook_config, + {}, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, diff --git a/tests/unit/hooks/test_dispatch.py b/tests/unit/hooks/test_dispatch.py index a332109a..26df72d6 100644 --- a/tests/unit/hooks/test_dispatch.py +++ b/tests/unit/hooks/test_dispatch.py @@ -6,7 +6,7 @@ from flexmock import flexmock from borgmatic.hooks import dispatch as module -def hook_function(config, log_prefix, thing, value): +def hook_function(hook_config, config, log_prefix, thing, value): ''' This test function gets mocked out below. ''' @@ -14,98 +14,104 @@ def hook_function(config, log_prefix, thing, value): def test_call_hook_invokes_module_function_with_arguments_and_returns_value(): - hooks = {'super_hook': flexmock(), 'other_hook': flexmock()} + config = {'super_hook': flexmock(), 'other_hook': flexmock()} expected_return_value = flexmock() test_module = sys.modules[__name__] flexmock(module).HOOK_NAME_TO_MODULE = {'super_hook': test_module} flexmock(test_module).should_receive('hook_function').with_args( - hooks['super_hook'], 'prefix', 55, value=66 + config['super_hook'], config, 'prefix', 55, value=66 ).and_return(expected_return_value).once() - return_value = module.call_hook('hook_function', hooks, 'prefix', 'super_hook', 55, value=66) + return_value = module.call_hook('hook_function', config, 'prefix', 'super_hook', 55, value=66) assert return_value == expected_return_value def test_call_hook_without_hook_config_invokes_module_function_with_arguments_and_returns_value(): - hooks = {'other_hook': flexmock()} + config = {'other_hook': flexmock()} expected_return_value = flexmock() test_module = sys.modules[__name__] flexmock(module).HOOK_NAME_TO_MODULE = {'super_hook': test_module} flexmock(test_module).should_receive('hook_function').with_args( - {}, 'prefix', 55, value=66 + {}, config, 'prefix', 55, value=66 ).and_return(expected_return_value).once() - return_value = module.call_hook('hook_function', hooks, 'prefix', 'super_hook', 55, value=66) + return_value = module.call_hook('hook_function', config, 'prefix', 'super_hook', 55, value=66) assert return_value == expected_return_value def test_call_hook_without_corresponding_module_raises(): - hooks = {'super_hook': flexmock(), 'other_hook': flexmock()} + config = {'super_hook': flexmock(), 'other_hook': flexmock()} test_module = sys.modules[__name__] flexmock(module).HOOK_NAME_TO_MODULE = {'other_hook': test_module} flexmock(test_module).should_receive('hook_function').never() with pytest.raises(ValueError): - module.call_hook('hook_function', hooks, 'prefix', 'super_hook', 55, value=66) + module.call_hook('hook_function', config, 'prefix', 'super_hook', 55, value=66) def test_call_hooks_calls_each_hook_and_collects_return_values(): - hooks = {'super_hook': flexmock(), 'other_hook': flexmock()} + config = {'super_hook': flexmock(), 'other_hook': flexmock()} expected_return_values = {'super_hook': flexmock(), 'other_hook': flexmock()} flexmock(module).should_receive('call_hook').and_return( expected_return_values['super_hook'] ).and_return(expected_return_values['other_hook']) - return_values = module.call_hooks('do_stuff', hooks, 'prefix', ('super_hook', 'other_hook'), 55) + return_values = module.call_hooks( + 'do_stuff', config, 'prefix', ('super_hook', 'other_hook'), 55 + ) assert return_values == expected_return_values def test_call_hooks_calls_skips_return_values_for_missing_hooks(): - hooks = {'super_hook': flexmock()} + config = {'super_hook': flexmock()} expected_return_values = {'super_hook': flexmock()} flexmock(module).should_receive('call_hook').and_return(expected_return_values['super_hook']) - return_values = module.call_hooks('do_stuff', hooks, 'prefix', ('super_hook', 'other_hook'), 55) + return_values = module.call_hooks( + 'do_stuff', config, 'prefix', ('super_hook', 'other_hook'), 55 + ) assert return_values == expected_return_values def test_call_hooks_calls_skips_return_values_for_null_hooks(): - hooks = {'super_hook': flexmock(), 'other_hook': None} + config = {'super_hook': flexmock(), 'other_hook': None} expected_return_values = {'super_hook': flexmock()} flexmock(module).should_receive('call_hook').and_return(expected_return_values['super_hook']) - return_values = module.call_hooks('do_stuff', hooks, 'prefix', ('super_hook', 'other_hook'), 55) + return_values = module.call_hooks( + 'do_stuff', config, 'prefix', ('super_hook', 'other_hook'), 55 + ) assert return_values == expected_return_values def test_call_hooks_even_if_unconfigured_calls_each_hook_and_collects_return_values(): - hooks = {'super_hook': flexmock(), 'other_hook': flexmock()} + config = {'super_hook': flexmock(), 'other_hook': flexmock()} expected_return_values = {'super_hook': flexmock(), 'other_hook': flexmock()} flexmock(module).should_receive('call_hook').and_return( expected_return_values['super_hook'] ).and_return(expected_return_values['other_hook']) return_values = module.call_hooks_even_if_unconfigured( - 'do_stuff', hooks, 'prefix', ('super_hook', 'other_hook'), 55 + 'do_stuff', config, 'prefix', ('super_hook', 'other_hook'), 55 ) assert return_values == expected_return_values def test_call_hooks_even_if_unconfigured_calls_each_hook_configured_or_not_and_collects_return_values(): - hooks = {'other_hook': flexmock()} + config = {'other_hook': flexmock()} expected_return_values = {'super_hook': flexmock(), 'other_hook': flexmock()} flexmock(module).should_receive('call_hook').and_return( expected_return_values['super_hook'] ).and_return(expected_return_values['other_hook']) return_values = module.call_hooks_even_if_unconfigured( - 'do_stuff', hooks, 'prefix', ('super_hook', 'other_hook'), 55 + 'do_stuff', config, 'prefix', ('super_hook', 'other_hook'), 55 ) assert return_values == expected_return_values diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index 5c6977da..fd43507b 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -143,6 +143,7 @@ def test_ping_monitor_hits_ping_url_for_start_state(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.START, monitoring_log_level=1, @@ -160,6 +161,7 @@ def test_ping_monitor_hits_ping_url_for_finish_state(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.FINISH, monitoring_log_level=1, @@ -177,6 +179,7 @@ def test_ping_monitor_hits_ping_url_for_fail_state(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.FAIL, monitoring_log_level=1, @@ -194,6 +197,7 @@ def test_ping_monitor_hits_ping_url_for_log_state(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.LOG, monitoring_log_level=1, @@ -213,6 +217,7 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.FINISH, monitoring_log_level=1, @@ -230,6 +235,7 @@ def test_ping_monitor_skips_ssl_verification_when_verify_tls_false(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.FINISH, monitoring_log_level=1, @@ -247,6 +253,7 @@ def test_ping_monitor_executes_ssl_verification_when_verify_tls_true(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.FINISH, monitoring_log_level=1, @@ -261,6 +268,7 @@ def test_ping_monitor_dry_run_does_not_hit_ping_url(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.START, monitoring_log_level=1, @@ -275,6 +283,7 @@ def test_ping_monitor_does_not_hit_ping_url_when_states_not_matching(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.START, monitoring_log_level=1, @@ -291,6 +300,7 @@ def test_ping_monitor_hits_ping_url_when_states_matching(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.START, monitoring_log_level=1, @@ -308,6 +318,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.START, monitoring_log_level=1, @@ -329,6 +340,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.ping_monitor( hook_config, + {}, 'config.yaml', state=module.monitor.State.START, monitoring_log_level=1, diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 5ac8ce96..234201f4 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -22,7 +22,7 @@ def test_dump_databases_runs_mongodump_for_each_database(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes def test_dump_databases_with_dry_run_skips_mongodump(): @@ -34,7 +34,7 @@ def test_dump_databases_with_dry_run_skips_mongodump(): flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == [] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=True) == [] def test_dump_databases_runs_mongodump_with_hostname_and_port(): @@ -63,7 +63,7 @@ def test_dump_databases_runs_mongodump_with_hostname_and_port(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_dump_databases_runs_mongodump_with_username_and_password(): @@ -101,7 +101,7 @@ def test_dump_databases_runs_mongodump_with_username_and_password(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_dump_databases_runs_mongodump_with_directory_format(): @@ -118,7 +118,7 @@ def test_dump_databases_runs_mongodump_with_directory_format(): shell=True, ).and_return(flexmock()).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [] def test_dump_databases_runs_mongodump_with_options(): @@ -136,7 +136,7 @@ def test_dump_databases_runs_mongodump_with_options(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_dump_databases_runs_mongodumpall_for_all_databases(): @@ -154,7 +154,7 @@ def test_dump_databases_runs_mongodumpall_for_all_databases(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_restore_database_dump_runs_mongorestore(): @@ -172,8 +172,8 @@ def test_restore_database_dump_runs_mongorestore(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -196,8 +196,8 @@ def test_restore_database_dump_errors_on_multiple_database_config(): with pytest.raises(ValueError): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=flexmock(), connection_params={ @@ -236,8 +236,8 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -284,8 +284,8 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -340,8 +340,8 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -396,8 +396,8 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -424,8 +424,8 @@ def test_restore_database_dump_runs_mongorestore_with_options(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -462,8 +462,8 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -490,8 +490,8 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -512,8 +512,8 @@ def test_restore_database_dump_with_dry_run_skips_restore(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=True, extract_process=flexmock(), connection_params={ @@ -539,8 +539,8 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=None, connection_params={ diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index cdcddf5e..4b55be47 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -63,7 +63,7 @@ def test_dump_databases_dumps_each_database(): dry_run_label=object, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes def test_dump_databases_dumps_with_password(): @@ -84,7 +84,7 @@ def test_dump_databases_dumps_with_password(): dry_run_label=object, ).and_return(process).once() - assert module.dump_databases([database], 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases([database], {}, 'test.yaml', dry_run=False) == [process] def test_dump_databases_dumps_all_databases_at_once(): @@ -102,7 +102,7 @@ def test_dump_databases_dumps_all_databases_at_once(): dry_run_label=object, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_dump_databases_dumps_all_databases_separately_when_format_configured(): @@ -122,7 +122,7 @@ def test_dump_databases_dumps_all_databases_separately_when_format_configured(): dry_run_label=object, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes def test_database_names_to_dump_runs_mysql_with_list_options(): @@ -365,7 +365,7 @@ def test_dump_databases_errors_for_missing_all_databases(): flexmock(module).should_receive('database_names_to_dump').and_return(()) with pytest.raises(ValueError): - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) def test_dump_databases_does_not_error_for_missing_all_databases_with_dry_run(): @@ -376,7 +376,7 @@ def test_dump_databases_does_not_error_for_missing_all_databases_with_dry_run(): ) flexmock(module).should_receive('database_names_to_dump').and_return(()) - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == [] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=True) == [] def test_restore_database_dump_runs_mysql_to_restore(): @@ -393,8 +393,8 @@ def test_restore_database_dump_runs_mysql_to_restore(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -415,8 +415,8 @@ def test_restore_database_dump_errors_on_multiple_database_config(): with pytest.raises(ValueError): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=flexmock(), connection_params={ @@ -442,8 +442,8 @@ def test_restore_database_dump_runs_mysql_with_options(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -478,8 +478,8 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -505,8 +505,8 @@ def test_restore_database_dump_runs_mysql_with_username_and_password(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -553,8 +553,8 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -603,8 +603,8 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -623,8 +623,8 @@ def test_restore_database_dump_with_dry_run_skips_restore(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=True, extract_process=flexmock(), connection_params={ diff --git a/tests/unit/hooks/test_ntfy.py b/tests/unit/hooks/test_ntfy.py index 9731df7a..7bace524 100644 --- a/tests/unit/hooks/test_ntfy.py +++ b/tests/unit/hooks/test_ntfy.py @@ -44,6 +44,7 @@ def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, @@ -65,6 +66,7 @@ def test_ping_monitor_with_auth_hits_hosted_ntfy_on_fail(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, @@ -83,6 +85,7 @@ def test_ping_monitor_auth_with_no_username_warning(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, @@ -101,6 +104,7 @@ def test_ping_monitor_auth_with_no_password_warning(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, @@ -114,6 +118,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.START, monitoring_log_level=1, @@ -127,6 +132,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FINISH, monitoring_log_level=1, @@ -144,6 +150,7 @@ def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, @@ -157,6 +164,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, @@ -172,6 +180,7 @@ def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, @@ -189,6 +198,7 @@ def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.START, monitoring_log_level=1, @@ -207,6 +217,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, @@ -229,6 +240,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.ping_monitor( hook_config, + {}, 'config.yaml', borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, diff --git a/tests/unit/hooks/test_pagerduty.py b/tests/unit/hooks/test_pagerduty.py index 0fccae00..5a5ed12a 100644 --- a/tests/unit/hooks/test_pagerduty.py +++ b/tests/unit/hooks/test_pagerduty.py @@ -8,6 +8,7 @@ def test_ping_monitor_ignores_start_state(): module.ping_monitor( {'integration_key': 'abc123'}, + {}, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, @@ -20,6 +21,7 @@ def test_ping_monitor_ignores_finish_state(): module.ping_monitor( {'integration_key': 'abc123'}, + {}, 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, @@ -32,6 +34,7 @@ def test_ping_monitor_calls_api_for_fail_state(): module.ping_monitor( {'integration_key': 'abc123'}, + {}, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, @@ -44,6 +47,7 @@ def test_ping_monitor_dry_run_does_not_call_api(): module.ping_monitor( {'integration_key': 'abc123'}, + {}, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, @@ -59,6 +63,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.ping_monitor( {'integration_key': 'abc123'}, + {}, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, @@ -76,6 +81,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.ping_monitor( {'integration_key': 'abc123'}, + {}, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 7ba45847..d7416100 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -217,7 +217,7 @@ def test_dump_databases_runs_pg_dump_for_each_database(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes def test_dump_databases_raises_when_no_database_names_to_dump(): @@ -227,7 +227,7 @@ def test_dump_databases_raises_when_no_database_names_to_dump(): flexmock(module).should_receive('database_names_to_dump').and_return(()) with pytest.raises(ValueError): - module.dump_databases(databases, 'test.yaml', {}, dry_run=False) + module.dump_databases(databases, {}, 'test.yaml', dry_run=False) def test_dump_databases_does_not_raise_when_no_database_names_to_dump(): @@ -236,7 +236,7 @@ def test_dump_databases_does_not_raise_when_no_database_names_to_dump(): flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(()) - module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == [] + module.dump_databases(databases, {}, 'test.yaml', dry_run=True) == [] def test_dump_databases_with_duplicate_dump_skips_pg_dump(): @@ -253,7 +253,7 @@ def test_dump_databases_with_duplicate_dump_skips_pg_dump(): flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [] def test_dump_databases_with_dry_run_skips_pg_dump(): @@ -270,7 +270,7 @@ def test_dump_databases_with_dry_run_skips_pg_dump(): flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == [] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=True) == [] def test_dump_databases_runs_pg_dump_with_hostname_and_port(): @@ -306,7 +306,7 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_dump_databases_runs_pg_dump_with_username_and_password(): @@ -342,7 +342,7 @@ def test_dump_databases_runs_pg_dump_with_username_and_password(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_dump_databases_runs_pg_dump_with_directory_format(): @@ -373,7 +373,7 @@ def test_dump_databases_runs_pg_dump_with_directory_format(): extra_environment={'PGSSLMODE': 'disable'}, ).and_return(flexmock()).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [] def test_dump_databases_runs_pg_dump_with_options(): @@ -406,7 +406,7 @@ def test_dump_databases_runs_pg_dump_with_options(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_dump_databases_runs_pg_dumpall_for_all_databases(): @@ -428,7 +428,7 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_dump_databases_runs_non_default_pg_dump(): @@ -460,7 +460,7 @@ def test_dump_databases_runs_non_default_pg_dump(): run_to_completion=False, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process] def test_restore_database_dump_runs_pg_restore(): @@ -501,8 +501,8 @@ def test_restore_database_dump_runs_pg_restore(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -526,8 +526,8 @@ def test_restore_database_dump_errors_on_multiple_database_config(): with pytest.raises(ValueError): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=flexmock(), connection_params={ @@ -587,8 +587,8 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -646,8 +646,8 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -724,8 +724,8 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -802,8 +802,8 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -862,8 +862,8 @@ def test_restore_database_dump_runs_pg_restore_with_options(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -900,8 +900,8 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -943,8 +943,8 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -1007,8 +1007,8 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={ @@ -1030,8 +1030,8 @@ def test_restore_database_dump_with_dry_run_skips_restore(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=True, extract_process=flexmock(), connection_params={ @@ -1081,8 +1081,8 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=None, connection_params={ @@ -1136,8 +1136,8 @@ def test_restore_database_dump_with_schemas_restores_schemas(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=None, connection_params={ diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py index 33317372..f61ffc75 100644 --- a/tests/unit/hooks/test_sqlite.py +++ b/tests/unit/hooks/test_sqlite.py @@ -17,7 +17,7 @@ def test_dump_databases_logs_and_skips_if_dump_already_exists(): flexmock(module.dump).should_receive('create_parent_directory_for_dump').never() flexmock(module).should_receive('execute_command').never() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [] def test_dump_databases_dumps_each_database(): @@ -37,7 +37,7 @@ def test_dump_databases_dumps_each_database(): processes[1] ) - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes def test_dumping_database_with_non_existent_path_warns_and_dumps_database(): @@ -55,7 +55,7 @@ def test_dumping_database_with_non_existent_path_warns_and_dumps_database(): flexmock(module.dump).should_receive('create_parent_directory_for_dump') flexmock(module).should_receive('execute_command').and_return(processes[0]) - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes def test_dumping_database_with_name_all_warns_and_dumps_all_databases(): @@ -75,7 +75,7 @@ def test_dumping_database_with_name_all_warns_and_dumps_all_databases(): flexmock(module.dump).should_receive('create_parent_directory_for_dump') flexmock(module).should_receive('execute_command').and_return(processes[0]) - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes def test_dump_databases_does_not_dump_if_dry_run(): @@ -89,7 +89,7 @@ def test_dump_databases_does_not_dump_if_dry_run(): flexmock(module.dump).should_receive('create_parent_directory_for_dump').never() flexmock(module).should_receive('execute_command').never() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == [] + assert module.dump_databases(databases, {}, 'test.yaml', dry_run=True) == [] def test_restore_database_dump_restores_database(): @@ -110,8 +110,8 @@ def test_restore_database_dump_restores_database(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={'restore_path': None}, @@ -138,8 +138,8 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={'restore_path': 'cli/path/to/database'}, @@ -166,8 +166,8 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={'restore_path': None}, @@ -183,8 +183,8 @@ def test_restore_database_dump_does_not_restore_database_if_dry_run(): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=True, extract_process=extract_process, connection_params={'restore_path': None}, @@ -198,8 +198,8 @@ def test_restore_database_dump_raises_error_if_database_config_is_invalid(): with pytest.raises(ValueError): module.restore_database_dump( database_config, - 'test.yaml', {}, + 'test.yaml', dry_run=False, extract_process=extract_process, connection_params={'restore_path': None}, From ab351548d2533110280dd6b24760f6910e5f9a61 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Jul 2023 17:40:02 -0700 Subject: [PATCH 317/344] Fix (some) end-to-end tests (#721). --- borgmatic/hooks/cronhub.py | 4 +- borgmatic/hooks/cronitor.py | 4 +- borgmatic/hooks/healthchecks.py | 4 +- borgmatic/hooks/ntfy.py | 4 +- borgmatic/hooks/pagerduty.py | 4 +- tests/end-to-end/test_database.py | 211 +++++++++---------- tests/end-to-end/test_validate_config.py | 1 - tests/integration/hooks/test_healthchecks.py | 4 +- tests/unit/hooks/test_healthchecks.py | 10 +- 9 files changed, 118 insertions(+), 128 deletions(-) diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index 170f1916..bbdc19a8 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -14,7 +14,7 @@ MONITOR_STATE_TO_CRONHUB = { def initialize_monitor( - ping_url, config_filename, monitoring_log_level, dry_run + ping_url, config, config_filename, monitoring_log_level, dry_run ): # pragma: no cover ''' No initialization is necessary for this monitor. @@ -55,7 +55,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev def destroy_monitor( - ping_url_or_uuid, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index d57920cd..fe4cc1d8 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -14,7 +14,7 @@ MONITOR_STATE_TO_CRONITOR = { def initialize_monitor( - ping_url, config_filename, monitoring_log_level, dry_run + ping_url, config, config_filename, monitoring_log_level, dry_run ): # pragma: no cover ''' No initialization is necessary for this monitor. @@ -50,7 +50,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev def destroy_monitor( - ping_url_or_uuid, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index 511e9566..ae0772c5 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -70,7 +70,7 @@ def format_buffered_logs_for_payload(): return payload -def initialize_monitor(hook_config, config_filename, monitoring_log_level, dry_run): +def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): ''' Add a handler to the root logger that stores in memory the most recent logs emitted. That way, we can send them all to Healthchecks upon a finish or failure state. But skip this if the @@ -133,7 +133,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'{config_filename}: Healthchecks error: {error}') -def destroy_monitor(hook_config, config_filename, monitoring_log_level, dry_run): +def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): ''' Remove the monitor handler that was added to the root logger. This prevents the handler from getting reused by other instances of this monitor. diff --git a/borgmatic/hooks/ntfy.py b/borgmatic/hooks/ntfy.py index 50aa387a..abe976f0 100644 --- a/borgmatic/hooks/ntfy.py +++ b/borgmatic/hooks/ntfy.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) def initialize_monitor( - ping_url, config_filename, monitoring_log_level, dry_run + ping_url, config, config_filename, monitoring_log_level, dry_run ): # pragma: no cover ''' No initialization is necessary for this monitor. @@ -75,7 +75,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev def destroy_monitor( - ping_url_or_uuid, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/pagerduty.py b/borgmatic/hooks/pagerduty.py index 382a402f..aeeec97c 100644 --- a/borgmatic/hooks/pagerduty.py +++ b/borgmatic/hooks/pagerduty.py @@ -13,7 +13,7 @@ EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue' def initialize_monitor( - integration_key, config_filename, monitoring_log_level, dry_run + integration_key, config, config_filename, monitoring_log_level, dry_run ): # pragma: no cover ''' No initialization is necessary for this monitor. @@ -75,7 +75,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev def destroy_monitor( - ping_url_or_uuid, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 0a10339e..9209ec39 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -22,60 +22,57 @@ def write_configuration( storing database dumps, dump format (for PostgreSQL), and encryption passphrase. ''' config = f''' -location: - source_directories: - - {source_directory} - repositories: - - {repository_path} - borgmatic_source_directory: {borgmatic_source_directory} +source_directories: + - {source_directory} +repositories: + - {repository_path} +borgmatic_source_directory: {borgmatic_source_directory} -storage: - encryption_passphrase: "test" +encryption_passphrase: "test" -hooks: - postgresql_databases: - - name: test - hostname: postgresql - username: postgres - password: test - format: {postgresql_dump_format} - - name: all - hostname: postgresql - username: postgres - password: test - - name: all - format: custom - hostname: postgresql - username: postgres - password: test - mysql_databases: - - name: test - hostname: mysql - username: root - password: test - - name: all - hostname: mysql - username: root - password: test - - name: all - format: sql - hostname: mysql - username: root - password: test - mongodb_databases: - - name: test - hostname: mongodb - username: root - password: test - authentication_database: admin - format: {mongodb_dump_format} - - name: all - hostname: mongodb - username: root - password: test - sqlite_databases: - - name: sqlite_test - path: /tmp/sqlite_test.db +postgresql_databases: + - name: test + hostname: postgresql + username: postgres + password: test + format: {postgresql_dump_format} + - name: all + hostname: postgresql + username: postgres + password: test + - name: all + format: custom + hostname: postgresql + username: postgres + password: test +mysql_databases: + - name: test + hostname: mysql + username: root + password: test + - name: all + hostname: mysql + username: root + password: test + - name: all + format: sql + hostname: mysql + username: root + password: test +mongodb_databases: + - name: test + hostname: mongodb + username: root + password: test + authentication_database: admin + format: {mongodb_dump_format} + - name: all + hostname: mongodb + username: root + password: test +sqlite_databases: + - name: sqlite_test + path: /tmp/sqlite_test.db ''' with open(config_path, 'w') as config_file: @@ -96,51 +93,48 @@ def write_custom_restore_configuration( restore_username, restore_password and restore_path. ''' config = f''' -location: - source_directories: - - {source_directory} - repositories: - - {repository_path} - borgmatic_source_directory: {borgmatic_source_directory} +source_directories: + - {source_directory} +repositories: + - {repository_path} +borgmatic_source_directory: {borgmatic_source_directory} -storage: - encryption_passphrase: "test" +encryption_passphrase: "test" -hooks: - postgresql_databases: - - name: test - hostname: postgresql - username: postgres - password: test - format: {postgresql_dump_format} - restore_hostname: postgresql2 - restore_port: 5433 - restore_username: postgres2 - restore_password: test2 - mysql_databases: - - name: test - hostname: mysql - username: root - password: test - restore_hostname: mysql2 - restore_port: 3307 - restore_username: root - restore_password: test2 - mongodb_databases: - - name: test - hostname: mongodb - username: root - password: test - authentication_database: admin - format: {mongodb_dump_format} - restore_hostname: mongodb2 - restore_port: 27018 - restore_username: root2 - restore_password: test2 - sqlite_databases: - - name: sqlite_test - path: /tmp/sqlite_test.db - restore_path: /tmp/sqlite_test2.db +postgresql_databases: + - name: test + hostname: postgresql + username: postgres + password: test + format: {postgresql_dump_format} + restore_hostname: postgresql2 + restore_port: 5433 + restore_username: postgres2 + restore_password: test2 +mysql_databases: + - name: test + hostname: mysql + username: root + password: test + restore_hostname: mysql2 + restore_port: 3307 + restore_username: root + restore_password: test2 +mongodb_databases: + - name: test + hostname: mongodb + username: root + password: test + authentication_database: admin + format: {mongodb_dump_format} + restore_hostname: mongodb2 + restore_port: 27018 + restore_username: root2 + restore_password: test2 +sqlite_databases: + - name: sqlite_test + path: /tmp/sqlite_test.db + restore_path: /tmp/sqlite_test2.db ''' with open(config_path, 'w') as config_file: @@ -161,23 +155,20 @@ def write_simple_custom_restore_configuration( these options for PostgreSQL. ''' config = f''' -location: - source_directories: - - {source_directory} - repositories: - - {repository_path} - borgmatic_source_directory: {borgmatic_source_directory} +source_directories: + - {source_directory} +repositories: + - {repository_path} +borgmatic_source_directory: {borgmatic_source_directory} -storage: - encryption_passphrase: "test" +encryption_passphrase: "test" -hooks: - postgresql_databases: - - name: test - hostname: postgresql - username: postgres - password: test - format: {postgresql_dump_format} +postgresql_databases: + - name: test + hostname: postgresql + username: postgres + password: test + format: {postgresql_dump_format} ''' with open(config_path, 'w') as config_file: diff --git a/tests/end-to-end/test_validate_config.py b/tests/end-to-end/test_validate_config.py index 4b86da4a..85a2006c 100644 --- a/tests/end-to-end/test_validate_config.py +++ b/tests/end-to-end/test_validate_config.py @@ -38,5 +38,4 @@ def test_validate_config_command_with_show_flag_displays_configuration(): f'validate-borgmatic-config --config {config_path} --show'.split(' ') ).decode(sys.stdout.encoding) - assert 'location:' in output assert 'repositories:' in output diff --git a/tests/integration/hooks/test_healthchecks.py b/tests/integration/hooks/test_healthchecks.py index 0c37013b..687b4873 100644 --- a/tests/integration/hooks/test_healthchecks.py +++ b/tests/integration/hooks/test_healthchecks.py @@ -10,7 +10,7 @@ def test_destroy_monitor_removes_healthchecks_handler(): original_handlers = list(logger.handlers) logger.addHandler(module.Forgetful_buffering_handler(byte_capacity=100, log_level=1)) - module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock()) + module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock()) assert logger.handlers == original_handlers @@ -19,6 +19,6 @@ def test_destroy_monitor_without_healthchecks_handler_does_not_raise(): logger = logging.getLogger() original_handlers = list(logger.handlers) - module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock()) + module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock()) assert logger.handlers == original_handlers diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index fd43507b..e3ab4386 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -86,7 +86,7 @@ def test_initialize_monitor_creates_log_handler_with_ping_body_limit(): ).once() module.initialize_monitor( - {'ping_body_limit': ping_body_limit}, 'test.yaml', monitoring_log_level, dry_run=False + {'ping_body_limit': ping_body_limit}, {}, 'test.yaml', monitoring_log_level, dry_run=False ) @@ -99,7 +99,7 @@ def test_initialize_monitor_creates_log_handler_with_default_ping_body_limit(): monitoring_log_level, ).once() - module.initialize_monitor({}, 'test.yaml', monitoring_log_level, dry_run=False) + module.initialize_monitor({}, {}, 'test.yaml', monitoring_log_level, dry_run=False) def test_initialize_monitor_creates_log_handler_with_zero_ping_body_limit(): @@ -112,7 +112,7 @@ def test_initialize_monitor_creates_log_handler_with_zero_ping_body_limit(): ).once() module.initialize_monitor( - {'ping_body_limit': ping_body_limit}, 'test.yaml', monitoring_log_level, dry_run=False + {'ping_body_limit': ping_body_limit}, {}, 'test.yaml', monitoring_log_level, dry_run=False ) @@ -121,7 +121,7 @@ def test_initialize_monitor_creates_log_handler_when_send_logs_true(): flexmock(module).should_receive('Forgetful_buffering_handler').once() module.initialize_monitor( - {'send_logs': True}, 'test.yaml', monitoring_log_level=1, dry_run=False + {'send_logs': True}, {}, 'test.yaml', monitoring_log_level=1, dry_run=False ) @@ -130,7 +130,7 @@ def test_initialize_monitor_bails_when_send_logs_false(): flexmock(module).should_receive('Forgetful_buffering_handler').never() module.initialize_monitor( - {'send_logs': False}, 'test.yaml', monitoring_log_level=1, dry_run=False + {'send_logs': False}, {}, 'test.yaml', monitoring_log_level=1, dry_run=False ) From 7d18f5907912fe43bb9b34bd0fba0dc7ef119e10 Mon Sep 17 00:00:00 2001 From: Yoann Laissus Date: Sun, 9 Jul 2023 11:45:51 +0200 Subject: [PATCH 318/344] Add a config entry for BORG_CHECK_I_KNOW_WHAT_I_AM_DOING env var --- borgmatic/borg/environment.py | 15 +++++++++++++-- borgmatic/config/schema.yaml | 6 ++++++ tests/unit/borg/test_environment.py | 7 +++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 1b14369a..d32d07d7 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -11,11 +11,15 @@ OPTION_TO_ENVIRONMENT_VARIABLE = { 'temporary_directory': 'TMPDIR', } -DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = { +DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = { 'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK', 'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK', } +DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = { + 'check_i_know_what_i_am_doing': 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', +} + def make_environment(storage_config): ''' @@ -33,8 +37,15 @@ def make_environment(storage_config): for ( option_name, environment_variable_name, - ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items(): + ) in DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE.items(): value = storage_config.get(option_name, False) environment[environment_variable_name] = 'yes' if value else 'no' + for ( + option_name, + environment_variable_name, + ) in DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE.items(): + value = storage_config.get(option_name, False) + environment[environment_variable_name] = 'YES' if value else 'NO' + return environment diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3f1b3cb4..84b512ca 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -405,6 +405,12 @@ properties: Bypass Borg error about a previously unknown unencrypted repository. Defaults to false. example: true + check_i_know_what_i_am_doing: + type: boolean + description: | + Bypass Borg confirmation about check with repair option. + Defaults to false. + example: true extra_borg_options: type: object additionalProperties: false diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index 4cef39b9..203a6b05 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -25,6 +25,7 @@ def test_make_environment_without_configuration_should_only_set_default_environm assert environment == { 'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no', 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no', + 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING': 'NO', } @@ -34,6 +35,12 @@ def test_make_environment_with_relocated_repo_access_should_override_default(): assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes' +def test_make_environment_check_i_know_what_i_am_doing_should_override_default(): + environment = module.make_environment({'check_i_know_what_i_am_doing': True}) + + assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'YES' + + def test_make_environment_with_integer_variable_value(): environment = module.make_environment({'borg_files_cache_ttl': 40}) assert environment.get('BORG_FILES_CACHE_TTL') == '40' From a3f47a6418665a363d3fd765992c872b496b25c1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2023 09:38:28 -0700 Subject: [PATCH 319/344] Remove some sections from tests (#721). --- tests/end-to-end/test_database.py | 6 +++--- tests/integration/config/test_generate.py | 24 +++++++++++------------ 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index 9209ec39..f9c36214 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -25,7 +25,7 @@ def write_configuration( source_directories: - {source_directory} repositories: - - {repository_path} + - path: {repository_path} borgmatic_source_directory: {borgmatic_source_directory} encryption_passphrase: "test" @@ -96,7 +96,7 @@ def write_custom_restore_configuration( source_directories: - {source_directory} repositories: - - {repository_path} + - path: {repository_path} borgmatic_source_directory: {borgmatic_source_directory} encryption_passphrase: "test" @@ -158,7 +158,7 @@ def write_simple_custom_restore_configuration( source_directories: - {source_directory} repositories: - - {repository_path} + - path: {repository_path} borgmatic_source_directory: {borgmatic_source_directory} encryption_passphrase: "test" diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 5df1825a..e332b899 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -58,14 +58,13 @@ foo: - baz - quux -location: - repositories: - - one - - two +repositories: + - one + - two - # This comment should be kept. - # COMMENT_OUT - other: thing +# This comment should be kept. +# COMMENT_OUT +other: thing ''' # flake8: noqa @@ -75,13 +74,12 @@ location: # - baz # - quux -location: - repositories: - - one - - two +repositories: + - one + - two - # This comment should be kept. -# other: thing +# This comment should be kept. +# other: thing ''' assert module.comment_out_optional_configuration(config.strip()) == expected_config.strip() From 196a226a7e20639e750147b547dbd0ddef4ed8f4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2023 09:44:00 -0700 Subject: [PATCH 320/344] Add "check_i_know_what_i_am_doing" option to NEWS (#724). --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index f1ecea0d..535e5afe 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,8 @@ https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/ * #719: Fix an error when running "borg key export" through borgmatic. * #720: Fix an error when dumping a MySQL database and the "exclude_nodump" option is set. + * #724: Add "check_i_know_what_i_am_doing" option to bypass Borg confirmation prompt when running + "check --repair". * When merging two configuration files, error gracefully if the two files do not adhere to the same format. * BREAKING: Remove the deprecated (and silently ignored) "--successful" flag on the "list" action, From bd9d592560213f0c4d6ab8cb30ffb7fc81660988 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2023 10:20:51 -0700 Subject: [PATCH 321/344] Truncate long command error output. --- NEWS | 2 ++ borgmatic/commands/borgmatic.py | 27 ++++++++++++--------------- tests/unit/commands/test_borgmatic.py | 16 +--------------- 3 files changed, 15 insertions(+), 30 deletions(-) diff --git a/NEWS b/NEWS index 2831f29a..1638e083 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,8 @@ * All deprecated configuration option values now generate warning logs. * Remove the deprecated (and non-functional) "--excludes" flag in favor of excludes within configuration. + * Fix an error when logging too-long command output during error handling. Now, long command output + is truncated before logging. 1.7.15 * #326: Add configuration options and command-line flags for backing up a database from one diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a869594e..af87ad67 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -141,7 +141,6 @@ def run_configuration(config_filename, config, arguments): f'{repository.get("label", repository["path"])}: Error running actions for repository', error, levelno=logging.WARNING, - log_command_error_output=True, ) ) logger.warning( @@ -531,26 +530,24 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): return (configs, logs) -def log_record(suppress_log=False, **kwargs): +def log_record(**kwargs): ''' Create a log record based on the given makeLogRecord() arguments, one of which must be - named "levelno". Log the record (unless suppress log is set) and return it. + named "levelno". Log the record and return it. ''' record = logging.makeLogRecord(kwargs) - if suppress_log: - return record - logger.handle(record) + return record -def log_error_records( - message, error=None, levelno=logging.CRITICAL, log_command_error_output=False -): +MAX_CAPTURED_OUTPUT_LENGTH = 1000 + + +def log_error_records(message, error=None, levelno=logging.CRITICAL): ''' - Given error message text, an optional exception object, an optional log level, and whether to - log the error output of a CalledProcessError (if any), log error summary information and also - yield it as a series of logging.LogRecord instances. + Given error message text, an optional exception object, and an optional log level, log error + summary information and also yield it as a series of logging.LogRecord instances. Note that because the logs are yielded as a generator, logs won't get logged unless you consume the generator output. @@ -566,12 +563,12 @@ def log_error_records( except CalledProcessError as error: yield log_record(levelno=levelno, levelname=level_name, msg=message) if error.output: - # Suppress these logs for now and save full error output for the log summary at the end. + output = error.output.decode('utf-8') yield log_record( levelno=levelno, levelname=level_name, - msg=error.output, - suppress_log=not log_command_error_output, + msg=output[:MAX_CAPTURED_OUTPUT_LENGTH] + + ' ...' * (len(output) > MAX_CAPTURED_OUTPUT_LENGTH), ) yield log_record(levelno=levelno, levelname=level_name, msg=error) except (ValueError, OSError) as error: diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index e94bdd8c..2867a408 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -238,7 +238,6 @@ def test_run_configuration_retries_hard_error(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return([flexmock()]) error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( @@ -278,13 +277,11 @@ def test_run_configuration_retries_round_robin(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return([flexmock()]).ordered() foo_error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( @@ -314,13 +311,11 @@ def test_run_configuration_retries_one_passes(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return(flexmock()).ordered() error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( @@ -344,7 +339,6 @@ def test_run_configuration_retry_wait(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() @@ -352,7 +346,6 @@ def test_run_configuration_retry_wait(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(time).should_receive('sleep').with_args(20).and_return().ordered() @@ -360,7 +353,6 @@ def test_run_configuration_retry_wait(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(time).should_receive('sleep').with_args(30).and_return().ordered() @@ -389,13 +381,11 @@ def test_run_configuration_retries_timeout_multiple_repos(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError, levelno=logging.WARNING, - log_command_error_output=True, ).and_return([flexmock()]).ordered() # Sleep before retrying foo (and passing) @@ -826,10 +816,6 @@ def test_log_record_does_not_raise(): module.log_record(levelno=1, foo='bar', baz='quux') -def test_log_record_with_suppress_does_not_raise(): - module.log_record(levelno=1, foo='bar', baz='quux', suppress_log=True) - - def test_log_error_records_generates_output_logs_for_message_only(): flexmock(module).should_receive('log_record').replace_with(dict) @@ -843,7 +829,7 @@ def test_log_error_records_generates_output_logs_for_called_process_error(): flexmock(module.logger).should_receive('getEffectiveLevel').and_return(logging.WARNING) logs = tuple( - module.log_error_records('Error', subprocess.CalledProcessError(1, 'ls', 'error output')) + module.log_error_records('Error', subprocess.CalledProcessError(1, 'ls', b'error output')) ) assert {log['levelno'] for log in logs} == {logging.CRITICAL} From e8dbca9d6809f6f60730e38ee5608472124e223f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2023 11:16:18 -0700 Subject: [PATCH 322/344] Truncate long command output without assuming an encoded byte string. --- borgmatic/borg/create.py | 2 ++ borgmatic/borg/info.py | 1 + borgmatic/borg/list.py | 2 ++ borgmatic/borg/rinfo.py | 1 + borgmatic/borg/rlist.py | 5 ++++- borgmatic/borg/version.py | 1 + borgmatic/commands/borgmatic.py | 26 ++++++++++++++++++------- borgmatic/execute.py | 7 +++++-- tests/unit/borg/test_create.py | 7 ++++++- tests/unit/borg/test_info.py | 3 +++ tests/unit/borg/test_list.py | 2 ++ tests/unit/borg/test_rinfo.py | 3 +++ tests/unit/borg/test_rlist.py | 8 ++++++++ tests/unit/borg/test_version.py | 1 + tests/unit/commands/test_borgmatic.py | 28 ++++++++++++++++++++++++++- 15 files changed, 85 insertions(+), 12 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 018f447d..d778e565 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -292,6 +292,7 @@ def collect_special_file_paths( capture_stderr=True, working_directory=working_directory, extra_environment=borg_environment, + borg_local_path=local_path, ) paths = tuple( @@ -510,6 +511,7 @@ def create_archive( create_command, working_directory=working_directory, extra_environment=borg_environment, + borg_local_path=local_path, ) else: execute_command( diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index 9a8bdda2..3e596ca4 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -66,6 +66,7 @@ def display_archives_info( return execute_command_and_capture_output( full_command, extra_environment=environment.make_environment(config), + borg_local_path=local_path, ) else: execute_command( diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index b3db8e90..5a245d97 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -123,6 +123,7 @@ def capture_archive_listing( remote_path, ), extra_environment=borg_environment, + borg_local_path=local_path, ) .strip('\n') .split('\n') @@ -217,6 +218,7 @@ def list_archive( remote_path, ), extra_environment=borg_environment, + borg_local_path=local_path, ) .strip('\n') .split('\n') diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index a7ae8229..ab4197e6 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -54,6 +54,7 @@ def display_repository_info( return execute_command_and_capture_output( full_command, extra_environment=extra_environment, + borg_local_path=local_path, ) else: execute_command( diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index b532a6aa..b6ceca31 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -43,6 +43,7 @@ def resolve_archive_name( output = execute_command_and_capture_output( full_command, extra_environment=environment.make_environment(config), + borg_local_path=local_path, ) try: latest_archive = output.strip().splitlines()[-1] @@ -138,7 +139,9 @@ def list_repository( ) if rlist_arguments.json: - return execute_command_and_capture_output(main_command, extra_environment=borg_environment) + return execute_command_and_capture_output( + main_command, extra_environment=borg_environment, borg_local_path=local_path + ) else: execute_command( main_command, diff --git a/borgmatic/borg/version.py b/borgmatic/borg/version.py index feb677ad..9ded62a7 100644 --- a/borgmatic/borg/version.py +++ b/borgmatic/borg/version.py @@ -21,6 +21,7 @@ def local_borg_version(config, local_path='borg'): output = execute_command_and_capture_output( full_command, extra_environment=environment.make_environment(config), + borg_local_path=local_path, ) try: diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index af87ad67..8f61a0d3 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -141,6 +141,7 @@ def run_configuration(config_filename, config, arguments): f'{repository.get("label", repository["path"])}: Error running actions for repository', error, levelno=logging.WARNING, + log_command_error_output=True, ) ) logger.warning( @@ -530,24 +531,29 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): return (configs, logs) -def log_record(**kwargs): +def log_record(suppress_log=False, **kwargs): ''' Create a log record based on the given makeLogRecord() arguments, one of which must be - named "levelno". Log the record and return it. + named "levelno". Log the record (unless suppress log is set) and return it. ''' record = logging.makeLogRecord(kwargs) - logger.handle(record) + if suppress_log: + return record + logger.handle(record) return record MAX_CAPTURED_OUTPUT_LENGTH = 1000 -def log_error_records(message, error=None, levelno=logging.CRITICAL): +def log_error_records( + message, error=None, levelno=logging.CRITICAL, log_command_error_output=False +): ''' - Given error message text, an optional exception object, and an optional log level, log error - summary information and also yield it as a series of logging.LogRecord instances. + Given error message text, an optional exception object, an optional log level, and whether to + log the error output of a CalledProcessError (if any), log error summary information and also + yield it as a series of logging.LogRecord instances. Note that because the logs are yielded as a generator, logs won't get logged unless you consume the generator output. @@ -563,12 +569,18 @@ def log_error_records(message, error=None, levelno=logging.CRITICAL): except CalledProcessError as error: yield log_record(levelno=levelno, levelname=level_name, msg=message) if error.output: - output = error.output.decode('utf-8') + try: + output = error.output.decode('utf-8') + except (UnicodeDecodeError, AttributeError): + output = error.output + + # Suppress these logs for now and save full error output for the log summary at the end. yield log_record( levelno=levelno, levelname=level_name, msg=output[:MAX_CAPTURED_OUTPUT_LENGTH] + ' ...' * (len(output) > MAX_CAPTURED_OUTPUT_LENGTH), + suppress_log=True, ) yield log_record(levelno=levelno, levelname=level_name, msg=error) except (ValueError, OSError) as error: diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 39691dac..9238a0c3 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -241,13 +241,16 @@ def execute_command_and_capture_output( shell=False, extra_environment=None, working_directory=None, + borg_local_path=None, ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its output (stdout). If capture stderr is True, then capture and return stderr in addition to stdout. If shell is True, execute the command within a shell. If an extra environment dict is given, then use it to augment the current environment, and pass the result into the command. If - a working directory is given, use that as the present working directory when running the command. + a working directory is given, use that as the present working directory when running the + command. If a Borg local path is given, and the command matches it (regardless of arguments), + treat exit code 1 as a warning instead of an error. Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' @@ -264,7 +267,7 @@ def execute_command_and_capture_output( cwd=working_directory, ) except subprocess.CalledProcessError as error: - if exit_code_indicates_error(command, error.returncode): + if exit_code_indicates_error(command, error.returncode, borg_local_path): raise output = error.output diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 5b196d9b..dfea86e2 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -451,13 +451,14 @@ def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command(): capture_stderr=True, working_directory=None, extra_environment=None, + borg_local_path='borg', ).and_return('Processing files ...\n- /foo\n+ /bar\n- /baz').once() flexmock(module).should_receive('special_file').and_return(True) flexmock(module).should_receive('any_parent_directories').and_return(False) module.collect_special_file_paths( ('borg', 'create', '--exclude-nodump'), - local_path=None, + local_path='borg', working_directory=None, borg_environment=None, skip_directories=flexmock(), @@ -758,6 +759,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',), working_directory=None, extra_environment=None, + borg_local_path='borg', ) insert_logging_mock(logging.INFO) @@ -842,6 +844,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',), working_directory=None, extra_environment=None, + borg_local_path='borg', ) insert_logging_mock(logging.DEBUG) @@ -2235,6 +2238,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',), working_directory=None, extra_environment=None, + borg_local_path='borg', ).and_return('[]') json_output = module.create_archive( @@ -2277,6 +2281,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',), working_directory=None, extra_environment=None, + borg_local_path='borg', ).and_return('[]') json_output = module.create_archive( diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index 3e93bb73..107c4863 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -73,6 +73,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + borg_local_path='borg', ).and_return('[]') insert_logging_mock(logging.INFO) @@ -127,6 +128,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + borg_local_path='borg', ).and_return('[]') insert_logging_mock(logging.DEBUG) @@ -154,6 +156,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None, + borg_local_path='borg', ).and_return('[]') json_output = module.display_archives_info( diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 2f82b802..4d1d5346 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -429,6 +429,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', 'repo'), extra_environment=None, + borg_local_path='borg', ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').and_return( ('borg', 'list', 'repo::archive1') @@ -667,6 +668,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'rlist', '--repo', 'repo'), extra_environment=None, + borg_local_path='borg', ).and_return('archive1\narchive2').once() flexmock(module).should_receive('make_list_command').with_args( diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index 8628b9aa..ee5e5c24 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -97,6 +97,7 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + borg_local_path='borg', ).and_return('[]') insert_logging_mock(logging.INFO) @@ -153,6 +154,7 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + borg_local_path='borg', ).and_return('[]') insert_logging_mock(logging.DEBUG) @@ -181,6 +183,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_flag(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None, + borg_local_path='borg', ).and_return('[]') json_output = module.display_repository_info( diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index 65fcef1d..a61a837e 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -37,6 +37,7 @@ def test_resolve_archive_name_calls_borg_with_flags(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + borg_local_path='borg', ).and_return(expected_archive + '\n') assert ( @@ -57,6 +58,7 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + borg_local_path='borg', ).and_return(expected_archive + '\n') insert_logging_mock(logging.INFO) @@ -78,6 +80,7 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + borg_local_path='borg', ).and_return(expected_archive + '\n') insert_logging_mock(logging.DEBUG) @@ -99,6 +102,7 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + borg_local_path='borg1', ).and_return(expected_archive + '\n') assert ( @@ -120,6 +124,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + borg_local_path='borg', ).and_return(expected_archive + '\n') assert ( @@ -140,6 +145,7 @@ def test_resolve_archive_name_without_archives_raises(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + borg_local_path='borg', ).and_return('') with pytest.raises(ValueError): @@ -159,6 +165,7 @@ def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--log-json') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + borg_local_path='borg', ).and_return(expected_archive + '\n') assert ( @@ -180,6 +187,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None, + borg_local_path='borg', ).and_return(expected_archive + '\n') assert ( diff --git a/tests/unit/borg/test_version.py b/tests/unit/borg/test_version.py index a051f693..a00235a5 100644 --- a/tests/unit/borg/test_version.py +++ b/tests/unit/borg/test_version.py @@ -17,6 +17,7 @@ def insert_execute_command_and_capture_output_mock( flexmock(module).should_receive('execute_command_and_capture_output').with_args( command, extra_environment=None, + borg_local_path=borg_local_path, ).once().and_return(version_output) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 2867a408..1c44e281 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -238,6 +238,7 @@ def test_run_configuration_retries_hard_error(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return([flexmock()]) error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( @@ -277,11 +278,13 @@ def test_run_configuration_retries_round_robin(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return([flexmock()]).ordered() foo_error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( @@ -311,11 +314,13 @@ def test_run_configuration_retries_one_passes(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return(flexmock()).ordered() error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( @@ -339,6 +344,7 @@ def test_run_configuration_retry_wait(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() @@ -346,6 +352,7 @@ def test_run_configuration_retry_wait(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(time).should_receive('sleep').with_args(20).and_return().ordered() @@ -353,6 +360,7 @@ def test_run_configuration_retry_wait(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(time).should_receive('sleep').with_args(30).and_return().ordered() @@ -381,11 +389,13 @@ def test_run_configuration_retries_timeout_multiple_repos(): 'foo: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(module).should_receive('log_error_records').with_args( 'bar: Error running actions for repository', OSError, levelno=logging.WARNING, + log_command_error_output=True, ).and_return([flexmock()]).ordered() # Sleep before retrying foo (and passing) @@ -816,6 +826,10 @@ def test_log_record_does_not_raise(): module.log_record(levelno=1, foo='bar', baz='quux') +def test_log_record_with_suppress_does_not_raise(): + module.log_record(levelno=1, foo='bar', baz='quux', suppress_log=True) + + def test_log_error_records_generates_output_logs_for_message_only(): flexmock(module).should_receive('log_record').replace_with(dict) @@ -824,7 +838,7 @@ def test_log_error_records_generates_output_logs_for_message_only(): assert {log['levelno'] for log in logs} == {logging.CRITICAL} -def test_log_error_records_generates_output_logs_for_called_process_error(): +def test_log_error_records_generates_output_logs_for_called_process_error_with_bytes_ouput(): flexmock(module).should_receive('log_record').replace_with(dict) flexmock(module.logger).should_receive('getEffectiveLevel').and_return(logging.WARNING) @@ -836,6 +850,18 @@ def test_log_error_records_generates_output_logs_for_called_process_error(): assert any(log for log in logs if 'error output' in str(log)) +def test_log_error_records_generates_output_logs_for_called_process_error_with_string_ouput(): + flexmock(module).should_receive('log_record').replace_with(dict) + flexmock(module.logger).should_receive('getEffectiveLevel').and_return(logging.WARNING) + + logs = tuple( + module.log_error_records('Error', subprocess.CalledProcessError(1, 'ls', 'error output')) + ) + + assert {log['levelno'] for log in logs} == {logging.CRITICAL} + assert any(log for log in logs if 'error output' in str(log)) + + def test_log_error_records_generates_logs_for_value_error(): flexmock(module).should_receive('log_record').replace_with(dict) From 003d4eac9358998cfea165b2fa3f389fc5d1cee4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2023 11:41:43 -0700 Subject: [PATCH 323/344] Remove extra argument (#721). --- borgmatic/actions/restore.py | 1 - 1 file changed, 1 deletion(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 8112207a..a59554a2 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -112,7 +112,6 @@ def restore_single_database( {hook_name: [database]}, repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, - config, global_arguments.dry_run, extract_process, connection_params, From ecd9e62147db3bcfb266847d82b5194cbf0e5bd0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Jul 2023 00:37:36 -0700 Subject: [PATCH 324/344] Fix last end-to-end database test (#721). --- borgmatic/actions/restore.py | 3 +- borgmatic/hooks/mongodb.py | 26 ++++++--- borgmatic/hooks/mysql.py | 26 +++++---- borgmatic/hooks/postgresql.py | 29 ++++++---- borgmatic/hooks/sqlite.py | 28 ++++++---- tests/unit/hooks/test_mongodb.py | 57 +++++++++++-------- tests/unit/hooks/test_mysql.py | 57 +++++++------------ tests/unit/hooks/test_postgresql.py | 85 ++++++++++++----------------- tests/unit/hooks/test_sqlite.py | 27 +++++---- 9 files changed, 178 insertions(+), 160 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index a59554a2..06fd1b87 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -109,8 +109,9 @@ def restore_single_database( # Run a single database restore, consuming the extract stdout (if any). borgmatic.hooks.dispatch.call_hooks( 'restore_database_dump', - {hook_name: [database]}, + config, repository['path'], + database['name'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, global_arguments.dry_run, extract_process, diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index 3c91a183..c94a084d 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -100,24 +100,32 @@ def make_database_dump_pattern(databases, config, log_prefix, name=None): # pra def restore_database_dump( - database_config, config, log_prefix, dry_run, extract_process, connection_params + databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params ): ''' - Restore the given MongoDB database from an extract stream. The database is supplied as a - one-element sequence containing a dict describing the database, as per the configuration schema. - Use the configuration dict to construct the destination path and the given log prefix in any log - entries. If this is a dry run, then don't actually restore anything. Trigger the given active - extract process (an instance of subprocess.Popen) to produce output to consume. + Restore the given MongoDB database from an extract stream. The databases are supplied as a + sequence containing one dict describing each database (as per the configuration schema), but + only the database corresponding to the given database name is restored. Use the configuration + dict to construct the destination path and the given log prefix in any log entries. If this is a + dry run, then don't actually restore anything. Trigger the given active extract process (an + instance of subprocess.Popen) to produce output to consume. If the extract process is None, then restore the dump from the filesystem rather than from an extract stream. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - if len(database_config) != 1: - raise ValueError('The database configuration value is invalid') + try: + database = next( + database_config + for database_config in databases_config + if database_config.get('name') == database_name + ) + except StopIteration: + raise ValueError( + f'A database named "{database_name}" could not be found in the configuration' + ) - database = database_config[0] dump_filename = dump.make_database_dump_filename( make_dump_path(config), database['name'], database.get('hostname') ) diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 8ca2f6ba..a3b34f15 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -181,21 +181,27 @@ def make_database_dump_pattern(databases, config, log_prefix, name=None): # pra def restore_database_dump( - database_config, config, log_prefix, dry_run, extract_process, connection_params + databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params ): ''' - Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a - one-element sequence containing a dict describing the database, as per the configuration schema. - Use the given log prefix in any log entries. If this is a dry run, then don't actually restore - anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce - output to consume. + Restore the given MySQL/MariaDB database from an extract stream. The databases are supplied as a + sequence containing one dict describing each database (as per the configuration schema), but + only the database corresponding to the given database name is restored. Use the given log + prefix in any log entries. If this is a dry run, then don't actually restore anything. Trigger + the given active extract process (an instance of subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - if len(database_config) != 1: - raise ValueError('The database configuration value is invalid') - - database = database_config[0] + try: + database = next( + database_config + for database_config in databases_config + if database_config.get('name') == database_name + ) + except StopIteration: + raise ValueError( + f'A database named "{database_name}" could not be found in the configuration' + ) hostname = connection_params['hostname'] or database.get( 'restore_hostname', database.get('hostname') diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 7bef5a70..598b878c 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -202,14 +202,15 @@ def make_database_dump_pattern(databases, config, log_prefix, name=None): # pra def restore_database_dump( - database_config, config, log_prefix, dry_run, extract_process, connection_params + databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params ): ''' - Restore the given PostgreSQL database from an extract stream. The database is supplied as a - one-element sequence containing a dict describing the database, as per the configuration schema. - Use the given configuration dict to construct the destination path and the given log prefix in - any log entries. If this is a dry run, then don't actually restore anything. Trigger the given - active extract process (an instance of subprocess.Popen) to produce output to consume. + Restore the given PostgreSQL database from an extract stream. The databases are supplied as a + sequence containing one dict describing each database (as per the configuration schema), but + only the database corresponding to the given database name is restored. Use the given + configuration dict to construct the destination path and the given log prefix in any log + entries. If this is a dry run, then don't actually restore anything. Trigger the given active + extract process (an instance of subprocess.Popen) to produce output to consume. If the extract process is None, then restore the dump from the filesystem rather than from an extract stream. @@ -219,10 +220,16 @@ def restore_database_dump( ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - if len(database_config) != 1: - raise ValueError('The database configuration value is invalid') - - database = database_config[0] + try: + database = next( + database_config + for database_config in databases_config + if database_config.get('name') == database_name + ) + except StopIteration: + raise ValueError( + f'A database named "{database_name}" could not be found in the configuration' + ) hostname = connection_params['hostname'] or database.get( 'restore_hostname', database.get('hostname') @@ -262,7 +269,7 @@ def restore_database_dump( + (() if extract_process else (dump_filename,)) + tuple( itertools.chain.from_iterable(('--schema', schema) for schema in database['schemas']) - if database['schemas'] + if database.get('schemas') else () ) ) diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py index 109f253a..524318bc 100644 --- a/borgmatic/hooks/sqlite.py +++ b/borgmatic/hooks/sqlite.py @@ -84,22 +84,30 @@ def make_database_dump_pattern(databases, config, log_prefix, name=None): # pra def restore_database_dump( - database_config, config, log_prefix, dry_run, extract_process, connection_params + databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params ): ''' - Restore the given SQLite3 database from an extract stream. The database is supplied as a - one-element sequence containing a dict describing the database, as per the configuration schema. - Use the given log prefix in any log entries. If this is a dry run, then don't actually restore - anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce - output to consume. + Restore the given SQLite3 database from an extract stream. The databases are supplied as a + sequence containing one dict describing each database (as per the configuration schema), but + only the database corresponding to the given database name is restored. Use the given log prefix + in any log entries. If this is a dry run, then don't actually restore anything. Trigger the + given active extract process (an instance of subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - if len(database_config) != 1: - raise ValueError('The database configuration value is invalid') + try: + database = next( + database_config + for database_config in databases_config + if database_config.get('name') == database_name + ) + except StopIteration: + raise ValueError( + f'A database named "{database_name}" could not be found in the configuration' + ) - database_path = connection_params['restore_path'] or database_config[0].get( - 'restore_path', database_config[0].get('path') + database_path = connection_params['restore_path'] or database.get( + 'restore_path', database.get('path') ) logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}') diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 234201f4..a676b583 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -158,7 +158,7 @@ def test_dump_databases_runs_mongodumpall_for_all_databases(): def test_restore_database_dump_runs_mongorestore(): - database_config = [{'name': 'foo', 'schemas': None}] + databases_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -171,9 +171,10 @@ def test_restore_database_dump_runs_mongorestore(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -185,8 +186,8 @@ def test_restore_database_dump_runs_mongorestore(): ) -def test_restore_database_dump_errors_on_multiple_database_config(): - database_config = [{'name': 'foo'}, {'name': 'bar'}] +def test_restore_database_dump_errors_on_empty_databases_config(): + databases_config = [] flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') @@ -195,9 +196,10 @@ def test_restore_database_dump_errors_on_multiple_database_config(): with pytest.raises(ValueError): module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=flexmock(), connection_params={ @@ -210,7 +212,7 @@ def test_restore_database_dump_errors_on_multiple_database_config(): def test_restore_database_dump_runs_mongorestore_with_hostname_and_port(): - database_config = [ + databases_config = [ {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} ] extract_process = flexmock(stdout=flexmock()) @@ -235,9 +237,10 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -250,7 +253,7 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port(): def test_restore_database_dump_runs_mongorestore_with_username_and_password(): - database_config = [ + databases_config = [ { 'name': 'foo', 'username': 'mongo', @@ -283,9 +286,10 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -298,7 +302,7 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password(): def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): - database_config = [ + databases_config = [ { 'name': 'foo', 'username': 'mongo', @@ -339,9 +343,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -354,7 +359,7 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore(): - database_config = [ + databases_config = [ { 'name': 'foo', 'username': 'mongo', @@ -395,9 +400,10 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -410,7 +416,7 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ def test_restore_database_dump_runs_mongorestore_with_options(): - database_config = [{'name': 'foo', 'restore_options': '--harder', 'schemas': None}] + databases_config = [{'name': 'foo', 'restore_options': '--harder', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -423,9 +429,10 @@ def test_restore_database_dump_runs_mongorestore_with_options(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -438,7 +445,7 @@ def test_restore_database_dump_runs_mongorestore_with_options(): def test_restore_databases_dump_runs_mongorestore_with_schemas(): - database_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] + databases_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -461,9 +468,10 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -476,7 +484,7 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas(): def test_restore_database_dump_runs_psql_for_all_database_dump(): - database_config = [{'name': 'all', 'schemas': None}] + databases_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') @@ -489,9 +497,10 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='all', dry_run=False, extract_process=extract_process, connection_params={ @@ -504,16 +513,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_with_dry_run_skips_restore(): - database_config = [{'name': 'foo', 'schemas': None}] + databases_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename') flexmock(module).should_receive('execute_command_with_processes').never() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=True, extract_process=flexmock(), connection_params={ @@ -526,7 +536,7 @@ def test_restore_database_dump_with_dry_run_skips_restore(): def test_restore_database_dump_without_extract_process_restores_from_disk(): - database_config = [{'name': 'foo', 'format': 'directory', 'schemas': None}] + databases_config = [{'name': 'foo', 'format': 'directory', 'schemas': None}] flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path') @@ -538,9 +548,10 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=None, connection_params={ diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 4b55be47..19ab3cc7 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -380,7 +380,7 @@ def test_dump_databases_does_not_error_for_missing_all_databases_with_dry_run(): def test_restore_database_dump_runs_mysql_to_restore(): - database_config = [{'name': 'foo'}] + databases_config = [{'name': 'foo'}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -392,9 +392,10 @@ def test_restore_database_dump_runs_mysql_to_restore(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -406,30 +407,8 @@ def test_restore_database_dump_runs_mysql_to_restore(): ) -def test_restore_database_dump_errors_on_multiple_database_config(): - database_config = [{'name': 'foo'}, {'name': 'bar'}] - - flexmock(module).should_receive('execute_command_with_processes').never() - flexmock(module).should_receive('execute_command').never() - - with pytest.raises(ValueError): - module.restore_database_dump( - database_config, - {}, - 'test.yaml', - dry_run=False, - extract_process=flexmock(), - connection_params={ - 'hostname': None, - 'port': None, - 'username': None, - 'password': None, - }, - ) - - def test_restore_database_dump_runs_mysql_with_options(): - database_config = [{'name': 'foo', 'restore_options': '--harder'}] + databases_config = [{'name': 'foo', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -441,9 +420,10 @@ def test_restore_database_dump_runs_mysql_with_options(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -456,7 +436,7 @@ def test_restore_database_dump_runs_mysql_with_options(): def test_restore_database_dump_runs_mysql_with_hostname_and_port(): - database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + databases_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -477,9 +457,10 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -492,7 +473,7 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port(): def test_restore_database_dump_runs_mysql_with_username_and_password(): - database_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] + databases_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -504,9 +485,10 @@ def test_restore_database_dump_runs_mysql_with_username_and_password(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -519,7 +501,7 @@ def test_restore_database_dump_runs_mysql_with_username_and_password(): def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): - database_config = [ + databases_config = [ { 'name': 'foo', 'username': 'root', @@ -552,9 +534,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -567,7 +550,7 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore(): - database_config = [ + databases_config = [ { 'name': 'foo', 'username': 'root', @@ -602,9 +585,10 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -617,14 +601,15 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ def test_restore_database_dump_with_dry_run_skips_restore(): - database_config = [{'name': 'foo'}] + databases_config = [{'name': 'foo'}] flexmock(module).should_receive('execute_command_with_processes').never() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=True, extract_process=flexmock(), connection_params={ diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index d7416100..51235ffc 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -464,7 +464,7 @@ def test_dump_databases_runs_non_default_pg_dump(): def test_restore_database_dump_runs_pg_restore(): - database_config = [{'name': 'foo', 'schemas': None}] + databases_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -500,9 +500,10 @@ def test_restore_database_dump_runs_pg_restore(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -514,33 +515,8 @@ def test_restore_database_dump_runs_pg_restore(): ) -def test_restore_database_dump_errors_on_multiple_database_config(): - database_config = [{'name': 'foo'}, {'name': 'bar'}] - - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) - flexmock(module).should_receive('make_dump_path') - flexmock(module.dump).should_receive('make_database_dump_filename') - flexmock(module).should_receive('execute_command_with_processes').never() - flexmock(module).should_receive('execute_command').never() - - with pytest.raises(ValueError): - module.restore_database_dump( - database_config, - {}, - 'test.yaml', - dry_run=False, - extract_process=flexmock(), - connection_params={ - 'restore_hostname': None, - 'restore_port': None, - 'restore_username': None, - 'restore_password': None, - }, - ) - - def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): - database_config = [ + databases_config = [ {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} ] extract_process = flexmock(stdout=flexmock()) @@ -586,9 +562,10 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -601,7 +578,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port(): def test_restore_database_dump_runs_pg_restore_with_username_and_password(): - database_config = [ + databases_config = [ {'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None} ] extract_process = flexmock(stdout=flexmock()) @@ -645,9 +622,10 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -660,7 +638,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password(): def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): - database_config = [ + databases_config = [ { 'name': 'foo', 'hostname': 'database.example.org', @@ -723,9 +701,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -738,7 +717,7 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore(): - database_config = [ + databases_config = [ { 'name': 'foo', 'hostname': 'database.example.org', @@ -801,9 +780,10 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -816,7 +796,7 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ def test_restore_database_dump_runs_pg_restore_with_options(): - database_config = [ + databases_config = [ { 'name': 'foo', 'restore_options': '--harder', @@ -861,9 +841,10 @@ def test_restore_database_dump_runs_pg_restore_with_options(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -876,7 +857,7 @@ def test_restore_database_dump_runs_pg_restore_with_options(): def test_restore_database_dump_runs_psql_for_all_database_dump(): - database_config = [{'name': 'all', 'schemas': None}] + databases_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -899,9 +880,10 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='all', dry_run=False, extract_process=extract_process, connection_params={ @@ -914,7 +896,7 @@ def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_runs_psql_for_plain_database_dump(): - database_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}] + databases_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -942,9 +924,10 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -957,7 +940,7 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump(): def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): - database_config = [ + databases_config = [ { 'name': 'foo', 'pg_restore_command': 'docker exec mycontainer pg_restore', @@ -1006,9 +989,10 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=extract_process, connection_params={ @@ -1021,7 +1005,7 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql(): def test_restore_database_dump_with_dry_run_skips_restore(): - database_config = [{'name': 'foo', 'schemas': None}] + databases_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') @@ -1029,9 +1013,10 @@ def test_restore_database_dump_with_dry_run_skips_restore(): flexmock(module).should_receive('execute_command_with_processes').never() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=True, extract_process=flexmock(), connection_params={ @@ -1044,7 +1029,7 @@ def test_restore_database_dump_with_dry_run_skips_restore(): def test_restore_database_dump_without_extract_process_restores_from_disk(): - database_config = [{'name': 'foo', 'schemas': None}] + databases_config = [{'name': 'foo', 'schemas': None}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') @@ -1080,9 +1065,10 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=None, connection_params={ @@ -1095,7 +1081,7 @@ def test_restore_database_dump_without_extract_process_restores_from_disk(): def test_restore_database_dump_with_schemas_restores_schemas(): - database_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] + databases_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') @@ -1135,9 +1121,10 @@ def test_restore_database_dump_with_schemas_restores_schemas(): ).once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='foo', dry_run=False, extract_process=None, connection_params={ diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py index f61ffc75..761a0557 100644 --- a/tests/unit/hooks/test_sqlite.py +++ b/tests/unit/hooks/test_sqlite.py @@ -93,7 +93,7 @@ def test_dump_databases_does_not_dump_if_dry_run(): def test_restore_database_dump_restores_database(): - database_config = [{'path': '/path/to/database', 'name': 'database'}] + databases_config = [{'path': '/path/to/database', 'name': 'database'}, {'name': 'other'}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -109,9 +109,10 @@ def test_restore_database_dump_restores_database(): flexmock(module.os).should_receive('remove').once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='database', dry_run=False, extract_process=extract_process, connection_params={'restore_path': None}, @@ -119,7 +120,7 @@ def test_restore_database_dump_restores_database(): def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore(): - database_config = [ + databases_config = [ {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} ] extract_process = flexmock(stdout=flexmock()) @@ -137,9 +138,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for flexmock(module.os).should_receive('remove').once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='database', dry_run=False, extract_process=extract_process, connection_params={'restore_path': 'cli/path/to/database'}, @@ -147,7 +149,7 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore(): - database_config = [ + databases_config = [ {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} ] extract_process = flexmock(stdout=flexmock()) @@ -165,9 +167,10 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ flexmock(module.os).should_receive('remove').once() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='database', dry_run=False, extract_process=extract_process, connection_params={'restore_path': None}, @@ -175,31 +178,33 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_ def test_restore_database_dump_does_not_restore_database_if_dry_run(): - database_config = [{'path': '/path/to/database', 'name': 'database'}] + databases_config = [{'path': '/path/to/database', 'name': 'database'}] extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('execute_command_with_processes').never() flexmock(module.os).should_receive('remove').never() module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='database', dry_run=True, extract_process=extract_process, connection_params={'restore_path': None}, ) -def test_restore_database_dump_raises_error_if_database_config_is_invalid(): - database_config = [] +def test_restore_database_dump_raises_error_if_database_config_is_empty(): + databases_config = [] extract_process = flexmock(stdout=flexmock()) with pytest.raises(ValueError): module.restore_database_dump( - database_config, + databases_config, {}, 'test.yaml', + database_name='database', dry_run=False, extract_process=extract_process, connection_params={'restore_path': None}, From d2fa205476bd2b82d05423742484c9d71c8e9e44 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Jul 2023 19:42:14 -0700 Subject: [PATCH 325/344] Update documentation for section removal (#721). --- README.md | 63 +++--- ...movable-drive-or-an-intermittent-server.md | 30 +-- docs/how-to/backup-your-databases.md | 1 + docs/how-to/deal-with-very-large-backups.md | 13 +- docs/how-to/make-backups-redundant.md | 20 +- docs/how-to/make-per-application-backups.md | 193 ++++++++++-------- docs/how-to/set-up-backups.md | 13 +- docs/how-to/upgrade.md | 13 +- 8 files changed, 184 insertions(+), 162 deletions(-) diff --git a/README.md b/README.md index 78691195..9cd52108 100644 --- a/README.md +++ b/README.md @@ -16,50 +16,41 @@ The canonical home of borgmatic is at ht Here's an example configuration file: ```yaml -location: - # List of source directories to backup. - source_directories: - - /home - - /etc +# List of source directories to backup. +source_directories: + - /home + - /etc - # Paths of local or remote repositories to backup to. - repositories: - - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo - label: borgbase - - path: /var/lib/backups/local.borg - label: local +# Paths of local or remote repositories to backup to. +repositories: + - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo + label: borgbase + - path: /var/lib/backups/local.borg + label: local -retention: - # Retention policy for how many backups to keep. - keep_daily: 7 - keep_weekly: 4 - keep_monthly: 6 +# Retention policy for how many backups to keep. +keep_daily: 7 +keep_weekly: 4 +keep_monthly: 6 -consistency: - # List of checks to run to validate your backups. - checks: - - name: repository - - name: archives - frequency: 2 weeks +# List of checks to run to validate your backups. +checks: + - name: repository + - name: archives + frequency: 2 weeks -hooks: - # Custom preparation scripts to run. - before_backup: - - prepare-for-backup.sh +# Custom preparation scripts to run. +before_backup: + - prepare-for-backup.sh - # Databases to dump and include in backups. - postgresql_databases: - - name: users +# Databases to dump and include in backups. +postgresql_databases: + - name: users - # Third-party services to notify you if backups aren't happening. - healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c +# Third-party services to notify you if backups aren't happening. +healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c ``` -Want to see borgmatic in action? Check out the screencast. - - - borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). ## Integrations diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index 04ccbf79..6bcc8950 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -44,14 +44,16 @@ file](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/), say at `/etc/borgmatic.d/removable.yaml`: ```yaml -location: - source_directories: - - /home +source_directories: + - /home - repositories: - - path: /mnt/removable/backup.borg +repositories: + - path: /mnt/removable/backup.borg ``` +Prior to version 1.8.0 Put +these options in the `location:` section of your configuration. + Prior to version 1.7.10 Omit the `path:` portion of the `repositories` list. @@ -77,18 +79,20 @@ optionally using `before_actions` instead. You can imagine a similar check for the sometimes-online server case: ```yaml -location: - source_directories: - - /home +source_directories: + - /home - repositories: - - path: ssh://me@buddys-server.org/./backup.borg +repositories: + - path: ssh://me@buddys-server.org/./backup.borg -hooks: - before_backup: - - ping -q -c 1 buddys-server.org > /dev/null || exit 75 +before_backup: + - ping -q -c 1 buddys-server.org > /dev/null || exit 75 ``` +Prior to version 1.8.0 Put the +first two options in the `location:` section of your configuration and the +`before_backup` option within the `hooks:` section. + Prior to version 1.7.10 Omit the `path:` portion of the `repositories` list. diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index bf2c7b68..b8aeca98 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -196,6 +196,7 @@ it is a mandatory option there: ```yaml location: source_directories: [] + hooks: mysql_databases: - name: all diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index d6142619..48199b97 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -162,11 +162,13 @@ either for a single repository or for all repositories. Disabling all consistency checks looks like this: ```yaml -consistency: - checks: - - name: disabled +checks: + - name: disabled ``` +Prior to version 1.8.0 Put +this option in the `consistency:` section of your configuration. + Prior to version 1.6.2 `checks` was a plain list of strings without the `name:` part. For instance: @@ -181,9 +183,8 @@ you can keep running consistency checks, but only against a subset of the repositories: ```yaml -consistency: - check_repositories: - - path/of/repository_to_check.borg +check_repositories: + - path/of/repository_to_check.borg ``` Finally, you can override your configuration file's consistency checks, and diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md index 2a4b8121..6f4d868b 100644 --- a/docs/how-to/make-backups-redundant.md +++ b/docs/how-to/make-backups-redundant.md @@ -12,18 +12,20 @@ it. borgmatic supports this in its configuration by specifying multiple backup repositories. Here's an example: ```yaml -location: - # List of source directories to backup. - source_directories: - - /home - - /etc +# List of source directories to backup. +source_directories: + - /home + - /etc - # Paths of local or remote repositories to backup to. - repositories: - - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo - - path: /var/lib/backups/local.borg +# Paths of local or remote repositories to backup to. +repositories: + - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo + - path: /var/lib/backups/local.borg ``` +Prior to version 1.8.0 Put +these options in the `location:` section of your configuration. + Prior to version 1.7.10 Omit the `path:` portion of the `repositories` list. diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index fb815d87..cae2fa2e 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -74,14 +74,15 @@ and borgmatic uses that format to name any new archive it creates. For instance: ```yaml -storage: - ... - archive_name_format: home-directories-{now} +archive_name_format: home-directories-{now} ``` -This means that when borgmatic creates an archive, its name will start with -the string `home-directories-` and end with a timestamp for its creation time. -If `archive_name_format` is unspecified, the default is +Prior to version 1.8.0 Put +this option in the `storage:` section of your configuration. + +This example means that when borgmatic creates an archive, its name will start +with the string `home-directories-` and end with a timestamp for its creation +time. If `archive_name_format` is unspecified, the default is `{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a timestamp in a particular format. @@ -156,23 +157,28 @@ them. To achieve this, you can put fragments of common configuration options into a file, and then include or inline that file into one or more borgmatic configuration files. -Let's say that you want to include common retention configuration across all +Let's say that you want to include common consistency check configuration across all of your configuration files. You could do that in each configuration file with the following: ```yaml -location: - ... +repositories: + - path: repo.borg -retention: - !include /etc/borgmatic/common_retention.yaml +checks: + !include /etc/borgmatic/common_checks.yaml ``` -And then the contents of `common_retention.yaml` could be: +Prior to version 1.8.0 These +options were organized into sections like `location:` and `consistency:`. + +The contents of `common_checks.yaml` could be: ```yaml -keep_hourly: 24 -keep_daily: 7 +- name: repository + frequency: 3 weeks +- name: archives + frequency: 2 weeks ``` To prevent borgmatic from trying to load these configuration fragments by @@ -188,11 +194,11 @@ Note that this form of include must be a YAML value rather than a key. For example, this will not work: ```yaml -location: - ... +repositories: + - path: repo.borg # Don't do this. It won't work! -!include /etc/borgmatic/common_retention.yaml +!include /etc/borgmatic/common_checks.yaml ``` But if you do want to merge in a YAML key *and* its values, keep reading! @@ -203,45 +209,48 @@ But if you do want to merge in a YAML key *and* its values, keep reading! If you need to get even fancier and merge in common configuration options, you can perform a YAML merge of included configuration using the YAML `<<` key. For instance, here's an example of a main configuration file that pulls in -retention and consistency options via a single include: +retention and consistency checks options via a single include: ```yaml -<<: !include /etc/borgmatic/common.yaml +repositories: + - path: repo.borg -location: - ... +<<: !include /etc/borgmatic/common.yaml ``` This is what `common.yaml` might look like: ```yaml -retention: - keep_hourly: 24 - keep_daily: 7 +keep_hourly: 24 +keep_daily: 7 -consistency: - checks: - - name: repository +checks: + - name: repository + frequency: 3 weeks + - name: archives + frequency: 2 weeks ``` -Once this include gets merged in, the resulting configuration would have all -of the `location` options from the original configuration file *and* the -`retention` and `consistency` options from the include. +Prior to version 1.8.0 These +options were organized into sections like `retention:` and `consistency:`. -Prior to borgmatic version 1.6.0, when there's a section collision between the -local file and the merged include, the local file's section takes precedence. -So if the `retention` section appears in both the local file and the include -file, the included `retention` is ignored in favor of the local `retention`. -But see below about deep merge in version 1.6.0+. +Once this include gets merged in, the resulting configuration would have all +of the options from the original configuration file *and* the options from the +include. + +Prior to version 1.6.0 When the +same option appeared in both the local file and the merged include, the local +file's value took precedence—meaning the included value was ignored in favor +of the local one. But see below about deep merge in version 1.6.0+. Note that this `<<` include merging syntax is only for merging in mappings (configuration options and their values). But if you'd like to include a -single value directly, please see the section above about standard includes. +single value directly, please see the above about standard includes. Additionally, there is a limitation preventing multiple `<<` include merges -per section. So for instance, that means you can do one `<<` merge at the -global level, another `<<` within each configuration section, etc. (This is a -YAML limitation.) +per file or option value. So for instance, that means you can do one `<<` +merge at the global level, another `<<` within each nested option value, etc. +(This is a YAML limitation.) ### Deep merge @@ -342,8 +351,8 @@ includes. ### Shallow merge Even though deep merging is generally pretty handy for included files, -sometimes you want specific sections in the local file to take precedence over -included sections—without any merging occurring for them. +sometimes you want specific options in the local file to take precedence over +included options—without any merging occurring for them. New in version 1.7.12 That's where the `!retain` tag comes in. Whenever you're merging an included file @@ -357,37 +366,38 @@ on the `retention` mapping: ```yaml <<: !include /etc/borgmatic/common.yaml -location: - repositories: - - path: repo.borg +repositories: + - path: repo.borg -retention: !retain - keep_daily: 5 +checks: !retain + - name: repository ``` And `common.yaml` like this: ```yaml -location: - repositories: - - path: common.borg +repositories: + - path: common.borg -retention: - keep_hourly: 24 - keep_daily: 7 +checks: + - name: archives ``` -Once this include gets merged in, the resulting configuration will have a -`keep_daily` value of `5` and nothing else in the `retention` section. That's -because the `!retain` tag says to retain the local version of `retention` and -ignore any values coming in from the include. But because the `repositories` -list doesn't have a `!retain` tag, it still gets merged together to contain -both `common.borg` and `repo.borg`. +Prior to version 1.8.0 These +options were organized into sections like `location:` and `consistency:`. -The `!retain` tag can only be placed on mappings and lists, and it goes right -after the name of the option (and its colon) on the same line. The effects of -`!retain` are recursive, meaning that if you place a `!retain` tag on a -top-level mapping, even deeply nested values within it will not be merged. +Once this include gets merged in, the resulting configuration will have a +`checks` value with a name of `repository` and no other values. That's because +the `!retain` tag says to retain the local version of `checks` and ignore any +values coming in from the include. But because the `repositories` list doesn't +have a `!retain` tag, it still gets merged together to contain both +`common.borg` and `repo.borg`. + +The `!retain` tag can only be placed on mappings (keys/values) and lists, and +it goes right after the name of the option (and its colon) on the same line. +The effects of `!retain` are recursive, meaning that if you place a `!retain` +tag on a top-level mapping, even deeply nested values within it will not be +merged. Additionally, the `!retain` tag only works in a configuration file that also performs a merge include with `<<: !include`. It doesn't make sense within, @@ -434,43 +444,50 @@ Whatever the reason, you can override borgmatic configuration options at the command-line via the `--override` flag. Here's an example: ```bash -borgmatic create --override location.remote_path=/usr/local/bin/borg1 +borgmatic create --override remote_path=/usr/local/bin/borg1 ``` What this does is load your configuration files, and for each one, disregard -the configured value for the `remote_path` option in the `location` section, -and use the value of `/usr/local/bin/borg1` instead. +the configured value for the `remote_path` option, and use the value of +`/usr/local/bin/borg1` instead. -You can even override multiple values at once. For instance: +Prior to version 1.8.0 Don't +forget to specify the section (like `location:`) that any option is in. + +You can even override nested values or multiple values at once. For instance: ```bash -borgmatic create --override section.option1=value1 section.option2=value2 +borgmatic create --override parent_option.option1=value1 parent_option.option2=value2 ``` This will accomplish the same thing: ```bash -borgmatic create --override section.option1=value1 --override section.option2=value2 +borgmatic create --override parent_option.option1=value1 --override parent_option.option2=value2 ``` +Prior to version 1.8.0 Don't +forget to specify the section that an option is in. That looks like a prefix +on the option name, e.g. `location.repositories`. + Note that each value is parsed as an actual YAML string, so you can even set list values by using brackets. For instance: ```bash -borgmatic create --override location.repositories=[test1.borg,test2.borg] +borgmatic create --override repositories=[test1.borg,test2.borg] ``` Or even a single list element: ```bash -borgmatic create --override location.repositories=[/root/test.borg] +borgmatic create --override repositories=[/root/test.borg] ``` If your override value contains special YAML characters like colons, then you'll need quotes for it to parse correctly: ```bash -borgmatic create --override location.repositories="['user@server:test.borg']" +borgmatic create --override repositories="['user@server:test.borg']" ``` There is not currently a way to override a single element of a list without @@ -486,7 +503,9 @@ indentation and a leading dash.) Be sure to quote your overrides if they contain spaces or other characters that your shell may interpret. -An alternate to command-line overrides is passing in your values via [environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). +An alternate to command-line overrides is passing in your values via +[environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). ## Constant interpolation @@ -506,16 +525,19 @@ constants: user: foo archive_prefix: bar -location: - source_directories: - - /home/{user}/.config - - /home/{user}/.ssh - ... +source_directories: + - /home/{user}/.config + - /home/{user}/.ssh -storage: - archive_name_format: '{archive_prefix}-{now}' +... + +archive_name_format: '{archive_prefix}-{now}' ``` +Prior to version 1.8.0 Don't +forget to specify the section (like `location:` or `storage:`) that any option +is in. + In this example, when borgmatic runs, all instances of `{user}` get replaced with `foo` and all instances of `{archive-prefix}` get replaced with `bar-`. (And in this particular example, `{now}` doesn't get replaced with anything, @@ -523,14 +545,13 @@ but gets passed directly to Borg.) After substitution, the logical result looks something like this: ```yaml -location: - source_directories: - - /home/foo/.config - - /home/foo/.ssh - ... +source_directories: + - /home/foo/.config + - /home/foo/.ssh -storage: - archive_name_format: 'bar-{now}' +... + +archive_name_format: 'bar-{now}' ``` An alternate to constants is passing in your values via [environment diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 043817ad..0797153f 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -140,13 +140,14 @@ use the `--destination` flag, for instance: `--destination You should edit the configuration file to suit your needs, as the generated values are only representative. All options are optional except where -indicated, so feel free to ignore anything you don't need. +indicated, so feel free to ignore anything you don't need. Be sure to use +spaces rather than tabs for indentation; YAML does not allow tabs. -Note that the configuration file is organized into distinct sections, each -with a section name like `location:` or `storage:`. So take care that if you -uncomment a particular option, also uncomment its containing section name, or -else borgmatic won't recognize the option. Also be sure to use spaces rather -than tabs for indentation; YAML does not allow tabs. +Prior to version 1.8.0 The +configuration file was organized into distinct sections, each with a section +name like `location:` or `storage:`. So in older versions of borgmatic, take +care that if you uncomment a particular option, also uncomment its containing +section name—or else borgmatic won't recognize the option. You can get the same sample configuration file from the [configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/), the diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index be85880f..43a28cc0 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -131,20 +131,21 @@ Let's say your original borgmatic repository configuration file looks something like this: ```yaml -location: - repositories: - - path: original.borg +repositories: + - path: original.borg ``` +Prior to version 1.8.0 This +option was found in the `location:` section of your configuration. + Prior to version 1.7.10 Omit the `path:` portion of the `repositories` list. Change it to a new (not yet created) repository path: ```yaml -location: - repositories: - - path: upgraded.borg +repositories: + - path: upgraded.borg ``` Then, run the `rcreate` action (formerly `init`) to create that new Borg 2 From 5b991b88ddfaba63abe0d7ac93a6a0d8e6d4f650 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Jul 2023 19:58:49 -0700 Subject: [PATCH 326/344] Rewrite documentation navigation URLs when being run locally. --- docs/_data/borgmatic.js | 5 +++++ docs/_includes/layouts/main.njk | 2 +- docs/docker-compose.yaml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 docs/_data/borgmatic.js diff --git a/docs/_data/borgmatic.js b/docs/_data/borgmatic.js new file mode 100644 index 00000000..5d76ca01 --- /dev/null +++ b/docs/_data/borgmatic.js @@ -0,0 +1,5 @@ +module.exports = function() { + return { + environment: process.env.NODE_ENV || "development" + }; +}; diff --git a/docs/_includes/layouts/main.njk b/docs/_includes/layouts/main.njk index a7e5b665..81300bae 100644 --- a/docs/_includes/layouts/main.njk +++ b/docs/_includes/layouts/main.njk @@ -11,7 +11,7 @@ headerClass: elv-header-default {% set navPages = collections.all | eleventyNavigation %} {% macro renderNavListItem(entry) -%} - {{ entry.title }} + {{ entry.title }} {%- if entry.children.length -%}