diff --git a/NEWS b/NEWS index 7cbaa6793..b2a83ec70 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,16 @@ 2.0.0.dev0 + * TL;DR: More flexible, completely revamped command hooks. All config options settable on the + command-line. Config option defaults for many command-line flags. New "key import" and "recreate" + actions. Almost everything is backwards compatible. * #262: Add a "default_actions" option that supports disabling default actions when borgmatic is run without any command-line arguments. + * #303: Deprecate the "--override" flag in favor of direct command-line flags for every borgmatic + configuration option. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides + * #303: Add configuration options that serve as defaults for some (but not all) command-line + action flags. For example, each entry in "repositories:" now has an "encryption" option that + applies to the "repo-create" action, serving as a default for the "--encryption" flag. See the + documentation for more information: https://torsion.org/borgmatic/docs/reference/configuration/ * #345: Add a "key import" action to import a repository key from backup. * #422: Add home directory expansion to file-based and KeePassXC credential hooks. * #610: Add a "recreate" action for recreating archives, for instance for retroactively excluding @@ -26,7 +36,7 @@ * #1048: Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested directories that reside on separate devices/filesystems. * #1050: Fix a failure in the "spot" check when the archive contains a symlink. - * #1051: Add configuration filename to "Successfully ran configuration file" log message. + * #1051: Add configuration filename to the "Successfully ran configuration file" log message. 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 87119ca08..13023ff99 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -170,7 +170,7 @@ def filter_checks_on_frequency( if calendar.day_name[datetime_now().weekday()] not in days: logger.info( - f"Skipping {check} check due to day of the week; check only runs on {'/'.join(days)} (use --force to check anyway)" + f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)" ) filtered_checks.remove(check) continue @@ -372,7 +372,7 @@ def collect_spot_check_source_paths( borgmatic.borg.create.make_base_create_command( dry_run=True, repository_path=repository['path'], - config=config, + config=dict(config, list_details=True), patterns=borgmatic.actions.create.process_patterns( borgmatic.actions.create.collect_patterns(config), working_directory, @@ -382,7 +382,6 @@ def collect_spot_check_source_paths( borgmatic_runtime_directory=borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - list_files=True, stream_processes=stream_processes, ) ) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index a8ab6a6f0..551c680da 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -37,9 +37,7 @@ def run_compact( global_arguments, local_path=local_path, remote_path=remote_path, - progress=compact_arguments.progress, cleanup_commits=compact_arguments.cleanup_commits, - threshold=compact_arguments.threshold, ) else: # pragma: nocover logger.info('Skipping compact (only available/needed in Borg 1.2+)') diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 4a72b2cdd..520bcb1c9 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -119,7 +119,9 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): bootstrap_arguments.repository, archive_name, [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths], - config, + # Only add progress here and not the extract_archive() call above, because progress + # conflicts with extract_to_stdout. + dict(config, progress=bootstrap_arguments.progress or False), local_borg_version, global_arguments, local_path=bootstrap_arguments.local_path, @@ -127,5 +129,4 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): extract_to_stdout=False, destination_path=bootstrap_arguments.destination, strip_components=bootstrap_arguments.strip_components, - progress=bootstrap_arguments.progress, ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index ecc688b5d..a92af261c 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -289,6 +289,16 @@ def run_create( ): return + if config.get('list_details') and config.get('progress'): + raise ValueError( + 'With the create action, only one of --list/--files/list_details and --progress/progress can be used.' + ) + + if config.get('list_details') and create_arguments.json: + raise ValueError( + 'With the create action, only one of --list/--files/list_details and --json can be used.' + ) + logger.info(f'Creating archive{dry_run_label}') working_directory = borgmatic.config.paths.get_working_directory(config) @@ -327,10 +337,7 @@ def run_create( borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - progress=create_arguments.progress, - stats=create_arguments.stats, json=create_arguments.json, - list_files=create_arguments.list_files, stream_processes=stream_processes, ) diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index d5c6bacb6..f04b06ff3 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -43,6 +43,5 @@ def run_export_tar( local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, - list_files=export_tar_arguments.list_files, strip_components=export_tar_arguments.strip_components, ) diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 6e2e79003..feaaac810 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -45,5 +45,4 @@ def run_extract( remote_path=remote_path, destination_path=extract_arguments.destination, strip_components=extract_arguments.strip_components, - progress=extract_arguments.progress, ) diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index e6252e4ea..38d35922f 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -24,18 +24,38 @@ def run_repo_create( return logger.info('Creating repository') + + encryption_mode = repo_create_arguments.encryption_mode or repository.get('encryption') + + if not encryption_mode: + raise ValueError( + 'With the repo-create action, either the --encryption flag or the repository encryption option is required.' + ) + borgmatic.borg.repo_create.create_repository( global_arguments.dry_run, repository['path'], config, local_borg_version, global_arguments, - repo_create_arguments.encryption_mode, + encryption_mode, repo_create_arguments.source_repository, repo_create_arguments.copy_crypt_key, - repo_create_arguments.append_only, - repo_create_arguments.storage_quota, - repo_create_arguments.make_parent_dirs, + ( + repository.get('append_only') + if repo_create_arguments.append_only is None + else repo_create_arguments.append_only + ), + ( + repository.get('storage_quota') + if repo_create_arguments.storage_quota is None + else repo_create_arguments.storage_quota + ), + ( + repository.get('make_parent_directories') + if repo_create_arguments.make_parent_directories is None + else repo_create_arguments.make_parent_directories + ), local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 8a27d16bd..c96015159 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,7 +17,13 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' + if transfer_arguments.archive and config.get('match_archives'): + raise ValueError( + 'With the transfer action, only one of --archive and --match-archives/match_archives can be used.' + ) + logger.info('Transferring archives to repository') + borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 00a090a24..935c52c85 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -32,7 +32,7 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument if prefix else ( flags.make_match_archives_flags( - check_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -170,7 +170,7 @@ def check_archives( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags - + (('--progress',) if check_arguments.progress else ()) + + (('--progress',) if config.get('progress') else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) @@ -180,7 +180,7 @@ def check_archives( # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. output_file=( - DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None + DO_NOT_CAPTURE if check_arguments.repair or config.get('progress') else None ), environment=environment.make_environment(config), working_directory=working_directory, diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index fd248cbf1..b443e8c0d 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -15,9 +15,7 @@ def compact_segments( global_arguments, local_path='borg', remote_path=None, - progress=False, cleanup_commits=False, - threshold=None, ): ''' Given dry-run flag, a local or remote repository path, a configuration dict, and the local Borg @@ -26,6 +24,7 @@ def compact_segments( umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) extra_borg_options = config.get('extra_borg_options', {}).get('compact', '') + threshold = config.get('compact_threshold') full_command = ( (local_path, 'compact') @@ -33,7 +32,7 @@ def compact_segments( + (('--umask', str(umask)) if umask else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--cleanup-commits',) if cleanup_commits else ()) + (('--threshold', str(threshold)) if threshold else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 54a7f9ad4..de5115dcd 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -196,7 +196,7 @@ def check_all_root_patterns_exist(patterns): if missing_paths: raise ValueError( - f"Source directories / root pattern paths do not exist: {', '.join(missing_paths)}" + f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}" ) @@ -213,9 +213,7 @@ def make_base_create_command( borgmatic_runtime_directory, local_path='borg', remote_path=None, - progress=False, json=False, - list_files=False, stream_processes=None, ): ''' @@ -293,7 +291,7 @@ def make_base_create_command( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--list', '--filter', list_filter_flags) - if list_files and not json and not progress + if config.get('list_details') and not json and not config.get('progress') else () ) + (('--dry-run',) if dry_run else ()) @@ -361,10 +359,7 @@ def create_archive( borgmatic_runtime_directory, local_path='borg', remote_path=None, - progress=False, - stats=False, json=False, - list_files=False, stream_processes=None, ): ''' @@ -389,28 +384,26 @@ def create_archive( borgmatic_runtime_directory, local_path, remote_path, - progress, json, - list_files, stream_processes, ) if json: output_log_level = None - elif list_files or (stats and not dry_run): + elif config.get('list_details') or (config.get('statistics') and not dry_run): output_log_level = logging.ANSWER else: output_log_level = logging.INFO # The progress output isn't compatible with captured and logged output, as progress messes with # the terminal directly. - output_file = DO_NOT_CAPTURE if progress else None + output_file = DO_NOT_CAPTURE if config.get('progress') else None create_flags += ( (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ()) - + (('--stats',) if stats and not json and not dry_run else ()) + + (('--stats',) if config.get('statistics') and not json and not dry_run else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--json',) if json else ()) ) borg_exit_codes = config.get('borg_exit_codes') diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index d967582ce..63ee0ef8b 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -34,7 +34,7 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives) + + borgmatic.borg.flags.make_flags('list', config.get('list_details')) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) if delete_arguments.force @@ -48,9 +48,17 @@ def make_delete_command( local_borg_version=local_borg_version, default_archive_name_format='*', ) + + (('--stats',) if config.get('statistics') else ()) + borgmatic.borg.flags.make_flags_from_arguments( delete_arguments, - excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'), + excludes=( + 'list_details', + 'statistics', + 'force', + 'match_archives', + 'archive', + 'repository', + ), ) + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version) ) @@ -98,7 +106,7 @@ def delete_archives( repo_delete_arguments = argparse.Namespace( repository=repository['path'], - list_archives=delete_arguments.list_archives, + list_details=delete_arguments.list_details, force=delete_arguments.force, cache_only=delete_arguments.cache_only, keep_security_info=delete_arguments.keep_security_info, diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index d8283535c..224c07b16 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -20,7 +20,6 @@ def export_tar_archive( local_path='borg', remote_path=None, tar_filter=None, - list_files=False, strip_components=None, ): ''' @@ -43,7 +42,7 @@ def export_tar_archive( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) - + (('--list',) if list_files else ()) + + (('--list',) if config.get('list_details') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) @@ -57,7 +56,7 @@ def export_tar_archive( + (tuple(paths) if paths else ()) ) - if list_files: + if config.get('list_details'): output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 52d7c35f8..aa354cbcf 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -77,7 +77,6 @@ def extract_archive( remote_path=None, destination_path=None, strip_components=None, - progress=False, extract_to_stdout=False, ): ''' @@ -92,8 +91,8 @@ def extract_archive( umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) - if progress and extract_to_stdout: - raise ValueError('progress and extract_to_stdout cannot both be set') + if config.get('progress') and extract_to_stdout: + raise ValueError('progress and extract to stdout cannot both be set') if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version): numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else () @@ -128,7 +127,7 @@ def extract_archive( + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( # Make the repository path absolute so the destination directory used below via changing @@ -148,7 +147,7 @@ def extract_archive( # The progress output isn't compatible with captured and logged output, as progress messes with # the terminal directly. - if progress: + if config.get('progress'): return execute_command( full_command, output_file=DO_NOT_CAPTURE, diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index bde80a972..2d90ef523 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -48,9 +48,7 @@ def make_info_command( if info_arguments.prefix else ( flags.make_match_archives_flags( - info_arguments.match_archives - or info_arguments.archive - or config.get('match_archives'), + info_arguments.archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 82a782015..34530eeb4 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -41,7 +41,7 @@ def make_prune_flags(config, prune_arguments, local_borg_version): if prefix else ( flags.make_match_archives_flags( - prune_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -77,7 +77,7 @@ def prune_archives( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--stats',) - if prune_arguments.stats + if config.get('statistics') and not dry_run and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version) else () @@ -85,16 +85,16 @@ def prune_archives( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + flags.make_flags_from_arguments( prune_arguments, - excludes=('repository', 'match_archives', 'stats', 'list_archives'), + excludes=('repository', 'match_archives', 'statistics', 'list_details'), ) - + (('--list',) if prune_arguments.list_archives else ()) + + (('--list',) if config.get('list_details') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) - if prune_arguments.stats or prune_arguments.list_archives: + if config.get('statistics') or config.get('list_details'): output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index c0ff07040..b687256ec 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -23,18 +23,16 @@ def recreate_archive( patterns=None, ): ''' - Given a local or remote repository path, an archive name, a configuration dict, - the local Borg version string, an argparse.Namespace of recreate arguments, - an argparse.Namespace of global arguments, optional local and remote Borg paths. - - Executes the recreate command with the given arguments. + Given a local or remote repository path, an archive name, a configuration dict, the local Borg + version string, an argparse.Namespace of recreate arguments, an argparse.Namespace of global + arguments, optional local and remote Borg paths, executes the recreate command with the given + arguments. ''' - lock_wait = config.get('lock_wait', None) exclude_flags = make_exclude_flags(config) compression = config.get('compression', None) chunker_params = config.get('chunker_params', None) - # Available recompress MODES: 'if-different' (default), 'always', 'never' + # Available recompress MODES: "if-different", "always", "never" (default) recompress = config.get('recompress', None) # Write patterns to a temporary file and use that file with --patterns-from. @@ -56,10 +54,10 @@ def recreate_archive( '--filter', make_list_filter_flags(local_borg_version, global_arguments.dry_run), ) - if recreate_arguments.list + if config.get('list_details') else () ) - # Flag --target works only for a single archive + # Flag --target works only for a single archive. + (('--target', recreate_arguments.target) if recreate_arguments.target and archive else ()) + ( ('--comment', shlex.quote(recreate_arguments.comment)) diff --git a/borgmatic/borg/repo_create.py b/borgmatic/borg/repo_create.py index b4cde3946..9ed2619e7 100644 --- a/borgmatic/borg/repo_create.py +++ b/borgmatic/borg/repo_create.py @@ -24,7 +24,7 @@ def create_repository( copy_crypt_key=False, append_only=None, storage_quota=None, - make_parent_dirs=False, + make_parent_directories=False, local_path='borg', remote_path=None, ): @@ -79,7 +79,7 @@ def create_repository( + (('--copy-crypt-key',) if copy_crypt_key else ()) + (('--append-only',) if append_only else ()) + (('--storage-quota', storage_quota) if storage_quota else ()) - + (('--make-parent-dirs',) if make_parent_dirs else ()) + + (('--make-parent-dirs',) if make_parent_directories else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + (('--log-json',) if global_arguments.log_json else ()) diff --git a/borgmatic/borg/repo_delete.py b/borgmatic/borg/repo_delete.py index fa66d3c05..dee5a1a9f 100644 --- a/borgmatic/borg/repo_delete.py +++ b/borgmatic/borg/repo_delete.py @@ -39,14 +39,14 @@ def make_repo_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', repo_delete_arguments.list_archives) + + borgmatic.borg.flags.make_flags('list', config.get('list_details')) + ( (('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ())) if repo_delete_arguments.force else () ) + borgmatic.borg.flags.make_flags_from_arguments( - repo_delete_arguments, excludes=('list_archives', 'force', 'repository') + repo_delete_arguments, excludes=('list_details', 'force', 'repository') ) + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version) ) diff --git a/borgmatic/borg/repo_list.py b/borgmatic/borg/repo_list.py index 9722758fe..9f05adbff 100644 --- a/borgmatic/borg/repo_list.py +++ b/borgmatic/borg/repo_list.py @@ -113,7 +113,7 @@ def make_repo_list_command( if repo_list_arguments.prefix else ( flags.make_match_archives_flags( - repo_list_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 3af998a6a..7b1680353 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -32,17 +32,22 @@ def transfer_archives( + flags.make_flags('remote-path', remote_path) + flags.make_flags('umask', config.get('umask')) + flags.make_flags('log-json', global_arguments.log_json) - + flags.make_flags('lock-wait', config.get('lock_wait', None)) + + flags.make_flags('lock-wait', config.get('lock_wait')) + + flags.make_flags('progress', config.get('progress')) + ( flags.make_flags_from_arguments( transfer_arguments, - excludes=('repository', 'source_repository', 'archive', 'match_archives'), + excludes=( + 'repository', + 'source_repository', + 'archive', + 'match_archives', + 'progress', + ), ) or ( flags.make_match_archives_flags( - transfer_arguments.match_archives - or transfer_arguments.archive - or config.get('match_archives'), + transfer_arguments.archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -56,7 +61,7 @@ def transfer_archives( return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None, + output_file=DO_NOT_CAPTURE if config.get('progress') else None, environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 15bb286c4..ec4041cbf 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,8 +1,13 @@ import collections +import io import itertools +import re import sys from argparse import ArgumentParser +import ruamel.yaml + +import borgmatic.config.schema from borgmatic.config import collect ACTION_ALIASES = { @@ -64,9 +69,9 @@ def get_subactions_for_actions(action_parsers): def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments): ''' - Given a sequence of string arguments and a dict from action name to parsed argparse.Namespace - arguments, return the string arguments with any values omitted that happen to be the same as - the name of a borgmatic action. + Given unparsed arguments as a sequence of strings and a dict from action name to parsed + argparse.Namespace arguments, return the string arguments with any values omitted that happen to + be the same as the name of a borgmatic action. This prevents, for instance, "check --only extract" from triggering the "extract" action. ''' @@ -283,17 +288,270 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -def make_parsers(): +OMITTED_FLAG_NAMES = {'match-archives', 'progress', 'statistics', 'list-details'} + + +def make_argument_description(schema, flag_name): ''' - Build a global arguments parser, individual action parsers, and a combined parser containing - both. Return them as a tuple. The global parser is useful for parsing just global arguments - while ignoring actions, and the combined parser is handy for displaying help that includes - everything: global flags, a list of actions, etc. + Given a configuration schema dict and a flag name for it, extend the schema's description with + an example or additional information as appropriate based on its type. Return the updated + description for use in a command-line argument. + ''' + description = schema.get('description') + schema_type = schema.get('type') + example = schema.get('example') + pieces = [description] if description else [] + + if '[0]' in flag_name: + pieces.append( + ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + ) + + if example and schema_type in ('array', 'object'): + example_buffer = io.StringIO() + yaml = ruamel.yaml.YAML(typ='safe') + yaml.default_flow_style = True + yaml.dump(example, example_buffer) + + pieces.append(f'Example value: "{example_buffer.getvalue().strip()}"') + + return ' '.join(pieces).replace('%', '%%') + + +def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): + r''' + Given an argparse._ArgumentGroup instance, a sequence of unparsed argument strings, and a dotted + flag name, add command-line array element flags that correspond to the given unparsed arguments. + + Here's the background. We want to support flags that can have arbitrary indices like: + + --foo.bar[1].baz + + But argparse doesn't support that natively because the index can be an arbitrary number. We + won't let that stop us though, will we? + + If the current flag name has an array component in it (e.g. a name with "[0]"), then make a + pattern that would match the flag name regardless of the number that's in it. The idea is that + we want to look for unparsed arguments that appear like the flag name, but instead of "[0]" they + have, say, "[1]" or "[123]". + + Next, we check each unparsed argument against that pattern. If one of them matches, add an + argument flag for it to the argument parser group. Example: + + Let's say flag_name is: + + --foo.bar[0].baz + + ... then the regular expression pattern will be: + + ^--foo\.bar\[\d+\]\.baz + + ... and, if that matches an unparsed argument of: + + --foo.bar[1].baz + + ... then an argument flag will get added equal to that unparsed argument. And so the unparsed + argument will match it when parsing is performed! In this manner, we're using the actual user + CLI input to inform what exact flags we support. + ''' + if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: + return + + pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + + try: + # Find an existing list index flag (and its action) corresponding to the given flag name. + (argument_action, existing_flag_name) = next( + (action, action_flag_name) + for action in arguments_group._group_actions + for action_flag_name in action.option_strings + if pattern.match(action_flag_name) + if f'--{flag_name}'.startswith(action_flag_name) + ) + + # Based on the type of the action (e.g. argparse._StoreTrueAction), look up the corresponding + # action registry name (e.g., "store_true") to pass to add_argument(action=...) below. + action_registry_name = next( + registry_name + for registry_name, action_type in arguments_group._registries['action'].items() + # Not using isinstance() here because we only want an exact match—no parent classes. + if type(argument_action) is action_type + ) + except StopIteration: + return + + for unparsed in unparsed_arguments: + unparsed_flag_name = unparsed.split('=', 1)[0] + destination_name = unparsed_flag_name.lstrip('-').replace('-', '_') + + if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name: + continue + + if action_registry_name in ('store_true', 'store_false'): + arguments_group.add_argument( + unparsed_flag_name, + action=action_registry_name, + default=argument_action.default, + dest=destination_name, + required=argument_action.nargs, + ) + else: + arguments_group.add_argument( + unparsed_flag_name, + action=action_registry_name, + choices=argument_action.choices, + default=argument_action.default, + dest=destination_name, + nargs=argument_action.nargs, + required=argument_action.nargs, + type=argument_action.type, + ) + + +def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): + ''' + Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of + unparsed argument strings, convert the entire schema into corresponding command-line flags and + add them to the arguments group. + + For instance, given a schema of: + + { + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer'} + } + } + } + } + + ... the following flag will be added to the arguments group: + + --foo.bar + + If "foo" is instead an array of objects, both of the following will get added: + + --foo + --foo[0].bar + + And if names are also passed in, they are considered to be the name components of an option + (e.g. "foo" and "bar") and are used to construct a resulting flag. + + Bail if the schema is not a dict. + ''' + if names is None: + names = () + + if not isinstance(schema, dict): + return + + schema_type = schema.get('type') + + # If this option has multiple types, just use the first one (that isn't "null"). + if isinstance(schema_type, list): + try: + schema_type = next(single_type for single_type in schema_type if single_type != 'null') + except StopIteration: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + # If this is an "object" type, recurse for each child option ("property"). + if schema_type == 'object': + properties = schema.get('properties') + + # If there are child properties, recurse for each one. But if there are no child properties, + # fall through so that a flag gets added below for the (empty) object. + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, child, unparsed_arguments, names + (name,) + ) + + return + + # If this is an "array" type, recurse for each items type child option. Don't return yet so that + # a flag also gets added below for the array itself. + if schema_type == 'array': + items = schema.get('items', {}) + properties = borgmatic.config.schema.get_properties(items) + + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, + child, + unparsed_arguments, + names[:-1] + (f'{names[-1]}[0]',) + (name,), + ) + # If there aren't any children, then this is an array of scalars. Recurse accordingly. + else: + add_arguments_from_schema( + arguments_group, items, unparsed_arguments, names[:-1] + (f'{names[-1]}[0]',) + ) + + flag_name = '.'.join(names).replace('_', '-') + + # Certain options already have corresponding flags on individual actions (like "create + # --progress"), so don't bother adding them to the global flags. + if not flag_name or flag_name in OMITTED_FLAG_NAMES: + return + + metavar = names[-1].upper() + description = make_argument_description(schema, flag_name) + + # The object=str and array=str given here is to support specifying an object or an array as a + # YAML string on the command-line. + argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) + + # As a UX nicety, add separate true and false flags for boolean options. + if schema_type == 'boolean': + arguments_group.add_argument( + f'--{flag_name}', + action='store_true', + default=None, + help=description, + ) + + if names[-1].startswith('no_'): + no_flag_name = '.'.join(names[:-1] + (names[-1][len('no_') :],)).replace('_', '-') + else: + no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-') + + arguments_group.add_argument( + f'--{no_flag_name}', + dest=flag_name.replace('-', '_'), + action='store_false', + default=None, + help=f'Set the --{flag_name} value to false.', + ) + else: + arguments_group.add_argument( + f'--{flag_name}', + type=argument_type, + metavar=metavar, + help=description, + ) + + add_array_element_arguments(arguments_group, unparsed_arguments, flag_name) + + +def make_parsers(schema, unparsed_arguments): + ''' + Given a configuration schema dict and unparsed arguments as a sequence of strings, build a + global arguments parser, individual action parsers, and a combined parser containing both. + Return them as a tuple. The global parser is useful for parsing just global arguments while + ignoring actions, and the combined parser is handy for displaying help that includes everything: + global flags, a list of actions, etc. ''' config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) - global_parser = ArgumentParser(add_help=False) + # Using allow_abbrev=False here prevents the global parser from erroring about "ambiguous" + # options like --encryption. Such options are intended for an action parser rather than the + # global parser, and so we don't want to error on them here. + global_parser = ArgumentParser(allow_abbrev=False, add_help=False) global_group = global_parser.add_argument_group('global arguments') global_group.add_argument( @@ -310,9 +568,6 @@ def make_parsers(): action='store_true', help='Go through the motions, but do not actually write to any repositories', ) - global_group.add_argument( - '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output' - ) global_group.add_argument( '-v', '--verbosity', @@ -389,6 +644,7 @@ def make_parsers(): action='store_true', help='Display installed version number of borgmatic and exit', ) + add_arguments_from_schema(global_group, schema, unparsed_arguments) global_plus_action_parser = ArgumentParser( description=''' @@ -416,7 +672,6 @@ def make_parsers(): '--encryption', dest='encryption_mode', help='Borg repository encryption mode', - required=True, ) repo_create_group.add_argument( '--source-repository', @@ -435,6 +690,7 @@ def make_parsers(): ) repo_create_group.add_argument( '--append-only', + default=None, action='store_true', help='Create an append-only repository', ) @@ -444,6 +700,8 @@ def make_parsers(): ) repo_create_group.add_argument( '--make-parent-dirs', + dest='make_parent_directories', + default=None, action='store_true', help='Create any missing parent directories of the repository directory', ) @@ -478,7 +736,7 @@ def make_parsers(): ) transfer_group.add_argument( '--progress', - default=False, + default=None, action='store_true', help='Display progress as each archive is transferred', ) @@ -545,13 +803,17 @@ def make_parsers(): ) prune_group.add_argument( '--stats', - dest='stats', - default=False, + dest='statistics', + default=None, action='store_true', help='Display statistics of the pruned archive [Borg 1 only]', ) prune_group.add_argument( - '--list', dest='list_archives', action='store_true', help='List archives kept/pruned' + '--list', + dest='list_details', + default=None, + action='store_true', + help='List archives kept/pruned', ) prune_group.add_argument( '--oldest', @@ -589,8 +851,7 @@ def make_parsers(): ) compact_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress as each segment is compacted', ) @@ -604,7 +865,7 @@ def make_parsers(): compact_group.add_argument( '--threshold', type=int, - dest='threshold', + dest='compact_threshold', help='Minimum saved space percentage threshold for compacting a segment, defaults to 10', ) compact_group.add_argument( @@ -625,20 +886,24 @@ def make_parsers(): ) create_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is backed up', ) create_group.add_argument( '--stats', - dest='stats', - default=False, + dest='statistics', + default=None, action='store_true', help='Display statistics of archive', ) create_group.add_argument( - '--list', '--files', dest='list_files', action='store_true', help='Show per-file details' + '--list', + '--files', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) create_group.add_argument( '--json', dest='json', default=False, action='store_true', help='Output results as JSON' @@ -659,8 +924,7 @@ def make_parsers(): ) check_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is checked', ) @@ -717,12 +981,15 @@ def make_parsers(): ) delete_group.add_argument( '--list', - dest='list_archives', + dest='list_details', + default=None, action='store_true', help='Show details for the deleted archives', ) delete_group.add_argument( '--stats', + dest='statistics', + default=None, action='store_true', help='Display statistics for the deleted archives', ) @@ -827,8 +1094,7 @@ def make_parsers(): ) extract_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is extracted', ) @@ -903,8 +1169,7 @@ def make_parsers(): ) config_bootstrap_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is extracted', ) @@ -997,7 +1262,12 @@ def make_parsers(): '--tar-filter', help='Name of filter program to pipe data through' ) export_tar_group.add_argument( - '--list', '--files', dest='list_files', action='store_true', help='Show per-file details' + '--list', + '--files', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) export_tar_group.add_argument( '--strip-components', @@ -1108,7 +1378,8 @@ def make_parsers(): ) repo_delete_group.add_argument( '--list', - dest='list_archives', + dest='list_details', + default=None, action='store_true', help='Show details for the archives in the given repository', ) @@ -1539,7 +1810,11 @@ def make_parsers(): help='Archive name, hash, or series to recreate', ) recreate_group.add_argument( - '--list', dest='list', action='store_true', help='Show per-file details' + '--list', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) recreate_group.add_argument( '--target', @@ -1595,15 +1870,18 @@ def make_parsers(): return global_parser, action_parsers, global_plus_action_parser -def parse_arguments(*unparsed_arguments): +def parse_arguments(schema, *unparsed_arguments): ''' - Given command-line arguments with which this script was invoked, parse the arguments and return - them as a dict mapping from action name (or "global") to an argparse.Namespace instance. + Given a configuration schema dict and the command-line arguments with which this script was + invoked and unparsed arguments as a sequence of strings, parse the arguments and return them as + a dict mapping from action name (or "global") to an argparse.Namespace instance. Raise ValueError if the arguments cannot be parsed. Raise SystemExit with an error code of 0 if "--help" was requested. ''' - global_parser, action_parsers, global_plus_action_parser = make_parsers() + global_parser, action_parsers, global_plus_action_parser = make_parsers( + schema, unparsed_arguments + ) arguments, remaining_action_arguments = parse_arguments_for_actions( unparsed_arguments, action_parsers.choices, global_parser ) @@ -1631,15 +1909,6 @@ def parse_arguments(*unparsed_arguments): f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}" ) - if 'create' in arguments and arguments['create'].list_files and arguments['create'].progress: - raise ValueError( - 'With the create action, only one of --list (--files) and --progress flags can be used.' - ) - if 'create' in arguments and arguments['create'].list_files and arguments['create'].json: - raise ValueError( - 'With the create action, only one of --list (--files) and --json flags can be used.' - ) - if ( ('list' in arguments and 'repo-info' in arguments and arguments['list'].json) or ('list' in arguments and 'info' in arguments and arguments['list'].json) @@ -1647,15 +1916,6 @@ def parse_arguments(*unparsed_arguments): ): raise ValueError('With the --json flag, multiple actions cannot be used together.') - if ( - 'transfer' in arguments - and arguments['transfer'].archive - and arguments['transfer'].match_archives - ): - raise ValueError( - 'With the transfer action, only one of --archive and --match-archives flags can be used.' - ) - if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives): raise ValueError( 'With the list action, only one of --prefix or --match-archives flags can be used.' diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index cd0b7190a..5bf41db0f 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -8,6 +8,8 @@ import time from queue import Queue from subprocess import CalledProcessError +import ruamel.yaml + import borgmatic.actions.borg import borgmatic.actions.break_lock import borgmatic.actions.change_passphrase @@ -35,6 +37,7 @@ import borgmatic.actions.restore import borgmatic.actions.transfer import borgmatic.commands.completion.bash import borgmatic.commands.completion.fish +import borgmatic.config.load import borgmatic.config.paths from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version @@ -597,14 +600,14 @@ def run_actions( ) -def load_configurations(config_filenames, overrides=None, resolve_env=True): +def load_configurations(config_filenames, arguments, overrides=None, resolve_env=True): ''' - Given a sequence of configuration filenames, a sequence of configuration file override strings - in the form of "option.suboption=value", and whether to resolve environment variables, load and - validate each configuration file. Return the results as a tuple of: dict of configuration - filename to corresponding parsed configuration, a sequence of paths for all loaded configuration - files (including includes), and a sequence of logging.LogRecord instances containing any parse - errors. + Given a sequence of configuration filenames, arguments as a dict from action name to + argparse.Namespace, a sequence of configuration file override strings in the form of + "option.suboption=value", and whether to resolve environment variables, load and validate each + configuration file. Return the results as a tuple of: dict of configuration filename to + corresponding parsed configuration, a sequence of paths for all loaded configuration files + (including includes), and a sequence of logging.LogRecord instances containing any parse errors. Log records are returned here instead of being logged directly because logging isn't yet initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this @@ -632,6 +635,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): configs[config_filename], paths, parse_logs = validate.parse_configuration( config_filename, validate.schema_filename(), + arguments, overrides, resolve_env, ) @@ -970,9 +974,17 @@ def check_and_show_help_on_no_args(configs): def main(extra_summary_logs=[]): # pragma: no cover configure_signals() configure_delayed_logging() + schema_filename = validate.schema_filename() try: - arguments = parse_arguments(*sys.argv[1:]) + schema = borgmatic.config.load.load_configuration(schema_filename) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: + configure_logging(logging.CRITICAL) + logger.critical(error) + exit_with_help_link() + + try: + arguments = parse_arguments(schema, *sys.argv[1:]) except ValueError as error: configure_logging(logging.CRITICAL) logger.critical(error) @@ -995,10 +1007,10 @@ def main(extra_summary_logs=[]): # pragma: no cover print(borgmatic.commands.completion.fish.fish_completion()) sys.exit(0) - validate = bool('validate' in arguments) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( config_filenames, + arguments, global_arguments.overrides, resolve_env=global_arguments.resolve_env and not validate, ) @@ -1013,7 +1025,7 @@ def main(extra_summary_logs=[]): # pragma: no cover any_json_flags = any( getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values() ) - color_enabled = should_do_markup(global_arguments.no_color or any_json_flags, configs) + color_enabled = should_do_markup(configs, any_json_flags) try: configure_logging( diff --git a/borgmatic/commands/completion/bash.py b/borgmatic/commands/completion/bash.py index 7bf28a429..c72fbbecc 100644 --- a/borgmatic/commands/completion/bash.py +++ b/borgmatic/commands/completion/bash.py @@ -1,5 +1,7 @@ import borgmatic.commands.arguments import borgmatic.commands.completion.actions +import borgmatic.commands.completion.flag +import borgmatic.config.validate def parser_flags(parser): @@ -7,7 +9,12 @@ def parser_flags(parser): Given an argparse.ArgumentParser instance, return its argument flags in a space-separated string. ''' - return ' '.join(option for action in parser._actions for option in action.option_strings) + return ' '.join( + flag_variant + for action in parser._actions + for flag_name in action.option_strings + for flag_variant in borgmatic.commands.completion.flag.variants(flag_name) + ) def bash_completion(): @@ -19,7 +26,10 @@ def bash_completion(): unused_global_parser, action_parsers, global_plus_action_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) global_flags = parser_flags(global_plus_action_parser) # Avert your eyes. diff --git a/borgmatic/commands/completion/fish.py b/borgmatic/commands/completion/fish.py index edca0226b..31b83e9cb 100644 --- a/borgmatic/commands/completion/fish.py +++ b/borgmatic/commands/completion/fish.py @@ -4,6 +4,7 @@ from textwrap import dedent import borgmatic.commands.arguments import borgmatic.commands.completion.actions +import borgmatic.config.validate def has_file_options(action: Action): @@ -26,9 +27,11 @@ def has_choice_options(action: Action): def has_unknown_required_param_options(action: Action): ''' A catch-all for options that take a required parameter, but we don't know what the parameter is. - This should be used last. These are actions that take something like a glob, a list of numbers, or a string. + This should be used last. These are actions that take something like a glob, a list of numbers, + or a string. - Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid. + Actions that match this pattern should not show the normal arguments, because those are unlikely + to be valid. ''' return ( action.required is True @@ -52,9 +55,9 @@ def has_exact_options(action: Action): def exact_options_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation that forces file completions, options completion, - or just that some value follow the action, if the action takes such an argument and was the last action on the - command line prior to the cursor. + Given an argparse.Action instance, return a completion invocation that forces file completions, + options completion, or just that some value follow the action, if the action takes such an + argument and was the last action on the command line prior to the cursor. Otherwise, return an empty string. ''' @@ -80,8 +83,9 @@ def exact_options_completion(action: Action): def dedent_strip_as_tuple(string: str): ''' - Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string. - Makes it easier to write multiline strings for completions when you join them with a tuple. + Dedent a string, then strip it to avoid requiring your first line to have content, then return a + tuple of the string. Makes it easier to write multiline strings for completions when you join + them with a tuple. ''' return (dedent(string).strip('\n'),) @@ -95,7 +99,10 @@ def fish_completion(): unused_global_parser, action_parsers, global_plus_action_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) all_action_parsers = ' '.join(action for action in action_parsers.choices.keys()) diff --git a/borgmatic/commands/completion/flag.py b/borgmatic/commands/completion/flag.py new file mode 100644 index 000000000..5a6bc7972 --- /dev/null +++ b/borgmatic/commands/completion/flag.py @@ -0,0 +1,13 @@ +def variants(flag_name): + ''' + Given a flag name as a string, yield it and any variations that should be complete-able as well. + For instance, for a string like "--foo[0].bar", yield "--foo[0].bar", "--foo[1].bar", ..., + "--foo[9].bar". + ''' + if '[0]' in flag_name: + for index in range(0, 10): + yield flag_name.replace('[0]', f'[{index}]') + + return + + yield flag_name diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py new file mode 100644 index 000000000..d5996f0a9 --- /dev/null +++ b/borgmatic/config/arguments.py @@ -0,0 +1,176 @@ +import io +import re + +import ruamel.yaml + +import borgmatic.config.schema + +LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+)\]$') + + +def set_values(config, keys, value): + ''' + Given a configuration dict, a sequence of parsed key strings, and a string value, descend into + the configuration hierarchy based on the given keys and set the value into the right place. + For example, consider these keys: + + ('foo', 'bar', 'baz') + + This looks up "foo" in the given configuration dict. And within that, it looks up "bar". And + then within that, it looks up "baz" and sets it to the given value. Another example: + + ('mylist[0]', 'foo') + + This looks for the zeroth element of "mylist" in the given configuration. And within that, it + looks up "foo" and sets it to the given value. + ''' + if not keys: + return + + first_key = keys[0] + + # Support "mylist[0]" list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(first_key) + + if match: + list_key = match.group('list_name') + list_index = int(match.group('index')) + + try: + if len(keys) == 1: + config[list_key][list_index] = value + + return + + if list_key not in config: + config[list_key] = [] + + set_values(config[list_key][list_index], keys[1:], value) + except (IndexError, KeyError): + raise ValueError(f'Argument list index {first_key} is out of range') + + return + + if len(keys) == 1: + config[first_key] = value + + return + + if first_key not in config: + config[first_key] = {} + + set_values(config[first_key], keys[1:], value) + + +def type_for_option(schema, option_keys): + ''' + Given a configuration schema dict and a sequence of keys identifying a potentially nested + option, e.g. ('extra_borg_options', 'create'), return the schema type of that option as a + string. + + Return None if the option or its type cannot be found in the schema. + ''' + option_schema = schema + + for key in option_keys: + # Support "name[0]"-style list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(key) + properties = borgmatic.config.schema.get_properties(option_schema) + + try: + if match: + option_schema = properties[match.group('list_name')]['items'] + else: + option_schema = properties[key] + except KeyError: + return None + + try: + return option_schema['type'] + except KeyError: + return None + + +def convert_value_type(value, option_type): + ''' + Given a string value and its schema type as a string, determine its logical type (string, + boolean, integer, etc.), and return it converted to that type. + + If the destination option type is a string, then leave the value as-is so that special + characters in it don't get interpreted as YAML during conversion. + + And if the source value isn't a string, return it as-is. + + Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML. + Raise ValueError if the parsed value doesn't match the option type. + ''' + if not isinstance(value, str): + return value + + if option_type == 'string': + return value + + try: + parsed_value = ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + except ruamel.yaml.error.YAMLError as error: + raise ValueError(f'Argument value "{value}" is invalid: {error.problem}') + + if not isinstance(parsed_value, borgmatic.config.schema.parse_type(option_type)): + raise ValueError(f'Argument value "{value}" is not of the expected type: {option_type}') + + return parsed_value + + +def prepare_arguments_for_config(global_arguments, schema): + ''' + Given global arguments as an argparse.Namespace and a configuration schema dict, parse each + argument that corresponds to an option in the schema and return a sequence of tuples (keys, + values) for that option, where keys is a sequence of strings. For instance, given the following + arguments: + + argparse.Namespace(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}) + + ... return this: + + ( + (('my_option', 'sub_option'), 'value1'), + (('other_option',), 'value2'), + ) + ''' + prepared_values = [] + + for argument_name, value in global_arguments.__dict__.items(): + if value is None: + continue + + keys = tuple(argument_name.split('.')) + option_type = type_for_option(schema, keys) + + # The argument doesn't correspond to any option in the schema, so ignore it. It's + # probably a flag that borgmatic has on the command-line but not in configuration. + if option_type is None: + continue + + prepared_values.append( + ( + keys, + convert_value_type(value, option_type), + ) + ) + + return tuple(prepared_values) + + +def apply_arguments_to_config(config, schema, arguments): + ''' + Given a configuration dict, a corresponding configuration schema dict, and arguments as a dict + from action name to argparse.Namespace, set those given argument values into their corresponding + configuration options in the configuration dict. + + This supports argument flags of the from "--foo.bar.baz" where each dotted component is a nested + configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list + element in the configuration. + ''' + for action_arguments in arguments.values(): + for keys, value in prepare_arguments_for_config(action_arguments, schema): + set_values(config, keys, value) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 58fd03a79..734457f23 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -1,11 +1,11 @@ import collections import io -import itertools import os import re import ruamel.yaml +import borgmatic.config.schema from borgmatic.config import load, normalize INDENT = 4 @@ -22,25 +22,7 @@ def insert_newline_before_comment(config, field_name): ) -def get_properties(schema): - ''' - Given a schema dict, return its properties. But if it's got sub-schemas with multiple different - potential properties, returned their merged properties instead (interleaved so the first - properties of each sub-schema come first). The idea is that the user should see all possible - options even if they're not all possible together. - ''' - if 'oneOf' in schema: - return dict( - item - for item in itertools.chain( - *itertools.zip_longest( - *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] - ) - ) - if item is not None - ) - - return schema['properties'] +SCALAR_SCHEMA_TYPES = {'string', 'boolean', 'integer', 'number'} def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False): @@ -54,37 +36,45 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i schema_type = schema.get('type') example = schema.get('example') - if example is not None: - return example - - if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): + if borgmatic.config.schema.compare_types(schema_type, {'array'}): config = ruamel.yaml.comments.CommentedSeq( - [ + example + if borgmatic.config.schema.compare_types( + schema['items'].get('type'), SCALAR_SCHEMA_TYPES + ) + else [ schema_to_sample_configuration( schema['items'], source_config, level, parent_is_sequence=True ) ] ) add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT)) - elif schema_type == 'object' or (isinstance(schema_type, list) and 'object' in schema_type): + elif borgmatic.config.schema.compare_types(schema_type, {'object'}): if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict): source_config = dict(collections.ChainMap(*source_config)) - config = ruamel.yaml.comments.CommentedMap( - [ - ( - field_name, - schema_to_sample_configuration( - sub_schema, (source_config or {}).get(field_name, {}), level + 1 - ), - ) - for field_name, sub_schema in get_properties(schema).items() - ] + config = ( + ruamel.yaml.comments.CommentedMap( + [ + ( + field_name, + schema_to_sample_configuration( + sub_schema, (source_config or {}).get(field_name, {}), level + 1 + ), + ) + for field_name, sub_schema in borgmatic.config.schema.get_properties( + schema + ).items() + ] + ) + or example ) indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) add_comments_to_configuration_object( config, schema, source_config, indent=indent, skip_first=parent_is_sequence ) + elif borgmatic.config.schema.compare_types(schema_type, SCALAR_SCHEMA_TYPES, match=all): + return example else: raise ValueError(f'Schema at level {level} is unsupported: {schema}') @@ -189,7 +179,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): return for field_name in config[0].keys(): - field_schema = get_properties(schema['items']).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema['items']).get(field_name, {}) description = field_schema.get('description') # No description to use? Skip it. @@ -223,7 +213,7 @@ def add_comments_to_configuration_object( if skip_first and index == 0: continue - field_schema = get_properties(schema).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema).get(field_name, {}) description = field_schema.get('description', '').strip() # If this isn't a default key, add an indicator to the comment flagging it to be commented diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 11f21ce09..f4199e6b4 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -326,7 +326,11 @@ def normalize(config_filename, config): config['repositories'] = [] for repository_dict in repositories: - repository_path = repository_dict['path'] + repository_path = repository_dict.get('path') + + if repository_path is None: + continue + if '~' in repository_path: logs.append( logging.makeLogRecord( diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 8a60cb50e..9067e6f99 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -1,7 +1,10 @@ import io +import logging import ruamel.yaml +logger = logging.getLogger(__name__) + def set_values(config, keys, value): ''' @@ -134,6 +137,11 @@ def apply_overrides(config, schema, raw_overrides): ''' overrides = parse_overrides(raw_overrides, schema) + if overrides: + logger.warning( + "The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set." + ) + for keys, value in overrides: set_values(config, keys, value) set_values(config, strip_section_names(keys), value) diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py new file mode 100644 index 000000000..3d13c0c25 --- /dev/null +++ b/borgmatic/config/schema.py @@ -0,0 +1,72 @@ +import decimal +import itertools + + +def get_properties(schema): + ''' + Given a schema dict, return its properties. But if it's got sub-schemas with multiple different + potential properties, return their merged properties instead (interleaved so the first + properties of each sub-schema come first). The idea is that the user should see all possible + options even if they're not all possible together. + ''' + if 'oneOf' in schema: + return dict( + item + for item in itertools.chain( + *itertools.zip_longest( + *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] + ) + ) + if item is not None + ) + + return schema.get('properties', {}) + + +SCHEMA_TYPE_TO_PYTHON_TYPE = { + 'array': list, + 'boolean': bool, + 'integer': int, + 'number': decimal.Decimal, + 'object': dict, + 'string': str, +} + + +def parse_type(schema_type, **overrides): + ''' + Given a schema type as a string, return the corresponding Python type. + + If any overrides are given in the from of a schema type string to a Python type, then override + the default type mapping with them. + + Raise ValueError if the schema type is unknown. + ''' + try: + return dict( + SCHEMA_TYPE_TO_PYTHON_TYPE, + **overrides, + )[schema_type] + except KeyError: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + +def compare_types(schema_type, target_types, match=any): + ''' + Given a schema type as a string or a list of strings (representing multiple types) and a set of + target type strings, return whether every schema type is in the set of target types. + + If the schema type is a list of strings, use the given match function (such as any or all) to + compare elements. For instance, if match is given as all, then every element of the schema_type + list must be in the target types. + ''' + if isinstance(schema_type, list): + if match(element_schema_type in target_types for element_schema_type in schema_type): + return True + + return False + + if schema_type in target_types: + return True + + return False diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 98923e905..aa56a8bea 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -33,13 +33,47 @@ properties: type: object required: - path + additionalProperties: false properties: path: type: string - example: ssh://user@backupserver/./{fqdn} + description: The local path or Borg URL of the repository. + example: ssh://user@backupserver/./sourcehostname.borg label: type: string + description: | + An optional label for the repository, used in logging + and to make selecting the repository easier on the + command-line. example: backupserver + encryption: + type: string + description: | + The encryption mode with which to create the repository, + only used for the repo-create action. To see the + available encryption modes, run "borg init --help" with + Borg 1 or "borg repo-create --help" with Borg 2. + example: repokey-blake2 + append_only: + type: boolean + description: | + Whether the repository should be created append-only, + only used for the repo-create action. Defaults to false. + example: true + storage_quota: + type: string + description: | + The storage quota with which to create the repository, + only used for the repo-create action. Defaults to no + quota. + example: 5G + make_parent_directories: + type: boolean + description: | + Whether any missing parent directories of the repository + path should be created, only used for the repo-create + action. Defaults to false. + example: true description: | A required list of local or remote repositories with paths and optional labels (which can be used with the --repository flag to @@ -48,8 +82,7 @@ properties: output of "borg help placeholders" for details. See ssh_command for SSH options like identity file or port. If systemd service is used, then add local repository paths in the systemd service file to the - ReadWritePaths list. Prior to borgmatic 1.7.10, repositories was a - list of plain path strings. + ReadWritePaths list. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver @@ -99,13 +132,13 @@ properties: used when backing up special devices such as /dev/zero. Defaults to false. But when a database hook is used, the setting here is ignored and read_special is considered true. - example: false + example: true flags: type: boolean description: | Record filesystem flags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true. - example: true + example: false files_cache: type: string description: | @@ -442,19 +475,19 @@ properties: type: boolean description: | Bypass Borg error about a repository that has been moved. Defaults - to not bypassing. + to false. example: true unknown_unencrypted_repo_access_is_ok: type: boolean description: | Bypass Borg error about a previously unknown unencrypted repository. - Defaults to not bypassing. + Defaults to false. example: true check_i_know_what_i_am_doing: type: boolean description: | Bypass Borg confirmation about check with repair option. Defaults to - an interactive prompt from Borg. + false and an interactive prompt from Borg. example: true extra_borg_options: type: object @@ -534,6 +567,12 @@ properties: not specified, borgmatic defaults to matching archives based on the archive_name_format (see above). example: sourcehostname + compact_threshold: + type: integer + description: | + Minimum saved space percentage threshold for compacting a segment, + defaults to 10. + example: 20 checks: type: array items: @@ -749,6 +788,10 @@ properties: List of one or more consistency checks to run on a periodic basis (if "frequency" is set) or every time borgmatic runs checks (if "frequency" is omitted). + example: + - name: archives + frequency: 2 weeks + - name: repository check_repositories: type: array items: @@ -770,9 +813,29 @@ properties: color: type: boolean description: | - Apply color to console output. Can be overridden with --no-color - command-line flag. Defaults to true. + Apply color to console output. Defaults to true. example: false + progress: + type: boolean + description: | + Display progress as each file or archive is processed when running + supported actions. Corresponds to the "--progress" flag on those + actions. Defaults to false. + example: true + statistics: + type: boolean + description: | + Display statistics for an archive when running supported actions. + Corresponds to the "--stats" flag on those actions. Defaults to + false. + example: true + list_details: + type: boolean + description: | + Display details for each file or archive as it is processed when + running supported actions. Corresponds to the "--list" flag on those + actions. Defaults to false. + example: true skip_actions: type: array items: @@ -1099,8 +1162,13 @@ properties: List of one or more command hooks to execute, triggered at particular points during borgmatic's execution. For each command hook, specify one of "before" or "after", not both. + example: + - before: action + when: [create] + run: [echo Backing up.] bootstrap: type: object + additionalProperties: false properties: store_config_files: type: boolean @@ -1313,6 +1381,9 @@ properties: https://www.postgresql.org/docs/current/app-pgdump.html and https://www.postgresql.org/docs/current/libpq-ssl.html for details. + example: + - name: users + hostname: database.example.org mariadb_databases: type: array items: @@ -1458,6 +1529,9 @@ properties: added to your source directories at runtime and streamed directly to Borg. Requires mariadb-dump/mariadb commands. See https://mariadb.com/kb/en/library/mysqldump/ for details. + example: + - name: users + hostname: database.example.org mysql_databases: type: array items: @@ -1603,6 +1677,9 @@ properties: to Borg. Requires mysqldump/mysql commands. See https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for details. + example: + - name: users + hostname: database.example.org sqlite_databases: type: array items: @@ -1650,6 +1727,15 @@ properties: sqlite3 version (e.g., one inside a running container). Defaults to "sqlite3". example: docker exec sqlite_container sqlite3 + description: | + List of one or more SQLite databases to dump before creating a + backup, run once per configuration file. The database dumps are + added to your source directories at runtime and streamed directly to + Borg. Requires the sqlite3 command. See https://sqlite.org/cli.html + for details. + example: + - name: users + path: /var/lib/db.sqlite mongodb_databases: type: array items: @@ -1771,6 +1857,9 @@ properties: to Borg. Requires mongodump/mongorestore commands. See https://docs.mongodb.com/database-tools/mongodump/ and https://docs.mongodb.com/database-tools/mongorestore/ for details. + example: + - name: users + hostname: database.example.org ntfy: type: object required: ['topic'] @@ -1807,6 +1896,7 @@ properties: example: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2 start: type: object + additionalProperties: false properties: title: type: string @@ -1830,6 +1920,7 @@ properties: example: incoming_envelope finish: type: object + additionalProperties: false properties: title: type: string @@ -1853,6 +1944,7 @@ properties: example: incoming_envelope fail: type: object + additionalProperties: false properties: title: type: string @@ -1911,6 +2003,7 @@ properties: example: hwRwoWsXMBWwgrSecfa9EfPey55WSN start: type: object + additionalProperties: false properties: message: type: string @@ -1950,8 +2043,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -1986,6 +2079,7 @@ properties: example: Pushover Link finish: type: object + additionalProperties: false properties: message: type: string @@ -2025,8 +2119,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -2061,6 +2155,7 @@ properties: example: Pushover Link fail: type: object + additionalProperties: false properties: message: type: string @@ -2100,8 +2195,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -2200,6 +2295,7 @@ properties: example: fakekey start: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2208,6 +2304,7 @@ properties: example: STARTED finish: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2216,6 +2313,7 @@ properties: example: FINISH fail: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2247,15 +2345,20 @@ properties: type: array items: type: object + additionalProperties: false required: - url - label properties: url: type: string + description: URL of this Apprise service. example: "gotify://hostname/token" label: type: string + description: | + Label used in borgmatic logs for this Apprise + service. example: gotify description: | A list of Apprise services to publish to with URLs and @@ -2270,7 +2373,7 @@ properties: send_logs: type: boolean description: | - Send borgmatic logs to Apprise services as part the + Send borgmatic logs to Apprise services as part of the "finish", "fail", and "log" states. Defaults to true. example: false logs_size_limit: @@ -2283,6 +2386,7 @@ properties: start: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2298,6 +2402,7 @@ properties: finish: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2313,6 +2418,7 @@ properties: fail: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2328,6 +2434,7 @@ properties: log: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2381,7 +2488,7 @@ properties: send_logs: type: boolean description: | - Send borgmatic logs to Healthchecks as part the "finish", + Send borgmatic logs to Healthchecks as part of the "finish", "fail", and "log" states. Defaults to true. example: false ping_body_limit: diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 0334e3019..9cb02243c 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -4,7 +4,7 @@ import os import jsonschema import ruamel.yaml -import borgmatic.config +import borgmatic.config.arguments from borgmatic.config import constants, environment, load, normalize, override @@ -21,6 +21,18 @@ def schema_filename(): return schema_path +def load_schema(schema_path): # pragma: no cover + ''' + Given a schema filename path, load the schema and return it as a dict. + + Raise Validation_error if the schema could not be parsed. + ''' + try: + return load.load_configuration(schema_path) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: + raise Validation_error(schema_path, (str(error),)) + + def format_json_error_path_element(path_element): ''' Given a path element into a JSON data structure, format it for display as a string. @@ -84,13 +96,17 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True): +def parse_configuration( + config_filename, schema_filename, arguments, overrides=None, resolve_env=True +): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML - rendition of JSON Schema format, a sequence of configuration file override strings in the form - of "option.suboption=value", and whether to resolve environment variables, return the parsed - configuration as a data structure of nested dicts and lists corresponding to the schema. Example - return value: + rendition of JSON Schema format, arguments as dict from action name to argparse.Namespace, a + sequence of configuration file override strings in the form of "option.suboption=value", and + whether to resolve environment variables, return the parsed configuration as a data structure of + nested dicts and lists corresponding to the schema. Example return value. + + Example return value: { 'source_directories': ['/home', '/etc'], @@ -113,6 +129,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) + borgmatic.config.arguments.apply_arguments_to_config(config, schema, arguments) override.apply_overrides(config, schema, overrides) constants.apply_constants(config, config.get('constants') if config else {}) diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 8e327b629..4eb34a34e 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -29,12 +29,13 @@ def interactive_console(): return sys.stderr.isatty() and os.environ.get('TERM') != 'dumb' -def should_do_markup(no_color, configs): +def should_do_markup(configs, json_enabled): ''' - Given the value of the command-line no-color argument, and a dict of configuration filename to - corresponding parsed configuration, determine if we should enable color marking up. + Given a dict of configuration filename to corresponding parsed configuration (which already have + any command-line overrides applied) and whether json is enabled, determine if we should enable + color marking up. ''' - if no_color: + if json_enabled: return False if any(config.get('color', True) is False for config in configs.values()): diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 7637fd725..3aecf478e 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -17,8 +17,8 @@ points as it runs. feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) instead.) -New in version 2.0.0 (not yet -released) Command hooks are now configured via a list of `commands:` in +New in version 2.0.0 (**not yet +released**) Command hooks are now configured via a list of `commands:` in your borgmatic configuration file. For example: ```yaml diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 095fef5f9..a81cd9432 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -482,16 +482,89 @@ applications, but then set the repository for each application at runtime. Or you might want to try a variant of an option for testing purposes without actually touching your configuration file. +New in version 2.0.0 Whatever the reason, you can override borgmatic configuration options at the -command-line via the `--override` flag. Here's an example: +command-line, as there's a command-line flag corresponding to every +configuration option (with its underscores converted to dashes). + +For instance, to override the `compression` configuration option, use the +corresponding `--compression` flag on the command-line: + +```bash +borgmatic create --compression zstd +``` + +What this does is load your given configuration files and for each one, disregard +the configured value for the `compression` option and use the value given on the +command-line instead—but just for the duration of the borgmatic run. + +You can override nested configuration options too by separating such option +names with a period. For instance: + +```bash +borgmatic create --bootstrap.store-config-files false +``` + +You can even set complex option data structures by using inline YAML syntax. For +example, set the `repositories` option with a YAML list of key/value pairs: + +```bash +borgmatic create --repositories "[{path: /mnt/backup, label: local}]" +``` + +If your override value contains characters like colons or spaces, then you'll +need to use quotes for it to parse correctly. + +You can also set individual nested options within existing list elements: + +```bash +borgmatic create --repositories[0].path /mnt/backup +``` + +This updates the `path` option for the first repository in `repositories`. +Change the `[0]` index as needed to address different list elements. And note +that this only works for elements already set in configuration; you can't append +new list elements from the command-line. + +See the [command-line reference +documentation](https://torsion.org/borgmatic/docs/reference/command-line/) for +the full set of available arguments, including examples of each for the complex +values. + +There are a handful of configuration options that don't have corresponding +command-line flags at the global scope, but instead have flags within individual +borgmatic actions. For instance, the `list_details` option can be overridden by +the `--list` flag that's only present on particular actions. Similarly with +`progress` and `--progress`, `statistics` and `--stats`, and `match_archives` +and `--match-archives`. + +Also note that if you want to pass a command-line flag itself as a value to one +of these override flags, that may not work. For instance, specifying +`--extra-borg-options.create --no-cache-sync` results in an error, because +`--no-cache-sync` gets interpreted as a borgmatic option (which in this case +doesn't exist) rather than a Borg option. + +An alternate to command-line overrides is passing in your values via +[environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). + + +### Deprecated overrides + +Prior to version 2.0.0 +Configuration overrides were performed with an `--override` flag. You can still +use `--override` with borgmatic 2.0.0+, but it's deprecated in favor of the new +command-line flags described above. + +Here's an example of `--override`: ```bash borgmatic create --override remote_path=/usr/local/bin/borg1 ``` -What this does is load your configuration files and for each one, disregard -the configured value for the `remote_path` option and use the value of -`/usr/local/bin/borg1` instead. +What this does is load your given configuration files and for each one, disregard +the configured value for the `remote_path` option and use the value given on the +command-line instead—but just for the duration of the borgmatic run. You can even override nested values or multiple values at once. For instance: @@ -540,10 +613,6 @@ reference](https://torsion.org/borgmatic/docs/reference/configuration/) for which options are list types. (YAML list values look like `- this` with an indentation and a leading dash.) -An alternate to command-line overrides is passing in your values via -[environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). - ## Constant interpolation diff --git a/tests/end-to-end/test_config_flag.py b/tests/end-to-end/test_config_flag.py new file mode 100644 index 000000000..279a1c6ec --- /dev/null +++ b/tests/end-to-end/test_config_flag.py @@ -0,0 +1,55 @@ +import os +import shlex +import shutil +import subprocess +import tempfile + + +def generate_configuration(config_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing (including injecting the given repository path and tacking on an encryption + passphrase). But don't actually set the repository path, as that's done on the command-line + below. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('- ssh://user@backupserver/./{fqdn}', '') # noqa: FS003 + .replace('- /var/local/backups/local.borg', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + 'encryption_passphrase: "test"' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_config_flags_do_not_error(): + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path) + + subprocess.check_call( + shlex.split( + f'borgmatic -v 2 --config {config_path} --repositories "[{{path: {repository_path}, label: repo}}]" repo-create --encryption repokey' + ) + ) + + subprocess.check_call( + shlex.split( + f'borgmatic create --config {config_path} --repositories[0].path "{repository_path}"' + ) + ) + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 4e9261e8d..80de7dcd7 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -53,7 +53,7 @@ def fuzz_argument(arguments, argument_name): def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments( - 'transfer', '--source-repository', 'foo' + {}, 'transfer', '--source-repository', 'foo' )['transfer'] flexmock(borgmatic.borg.transfer).should_receive('execute_command').replace_with( assert_command_does_not_duplicate_flags @@ -74,7 +74,7 @@ def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): def test_prune_archives_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('prune')['prune'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'prune')['prune'] flexmock(borgmatic.borg.prune).should_receive('execute_command').replace_with( assert_command_does_not_duplicate_flags ) @@ -94,7 +94,7 @@ def test_prune_archives_command_does_not_duplicate_flags_or_raise(): def test_mount_archive_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('mount', '--mount-point', 'tmp')[ + arguments = borgmatic.commands.arguments.parse_arguments({}, 'mount', '--mount-point', 'tmp')[ 'mount' ] flexmock(borgmatic.borg.mount).should_receive('execute_command').replace_with( @@ -116,7 +116,7 @@ def test_mount_archive_command_does_not_duplicate_flags_or_raise(): def test_make_list_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('list')['list'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'list')['list'] for argument_name in dir(arguments): if argument_name.startswith('_'): @@ -134,7 +134,7 @@ def test_make_list_command_does_not_duplicate_flags_or_raise(): def test_make_repo_list_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('repo-list')['repo-list'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'repo-list')['repo-list'] for argument_name in dir(arguments): if argument_name.startswith('_'): @@ -152,7 +152,7 @@ def test_make_repo_list_command_does_not_duplicate_flags_or_raise(): def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('info')['info'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'info')['info'] flexmock(borgmatic.borg.info).should_receive('execute_command_and_capture_output').replace_with( assert_command_does_not_duplicate_flags ) diff --git a/tests/integration/commands/completion/test_actions.py b/tests/integration/commands/completion/test_actions.py index 2e6fde9b4..52dfd268f 100644 --- a/tests/integration/commands/completion/test_actions.py +++ b/tests/integration/commands/completion/test_actions.py @@ -1,4 +1,5 @@ import borgmatic.commands.arguments +import borgmatic.config.validate from borgmatic.commands.completion import actions as module @@ -7,7 +8,10 @@ def test_available_actions_uses_only_subactions_for_action_with_subactions(): unused_global_parser, action_parsers, unused_combined_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) actions = module.available_actions(action_parsers, 'config') @@ -20,7 +24,10 @@ def test_available_actions_omits_subactions_for_action_without_subactions(): unused_global_parser, action_parsers, unused_combined_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) actions = module.available_actions(action_parsers, 'list') diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 4399c8678..b3ffef0d1 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -4,11 +4,144 @@ from flexmock import flexmock from borgmatic.commands import arguments as module +def test_make_argument_description_with_object_adds_example(): + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + 'example': {'bar': 'baz'}, + }, + flag_name='flag', + ) + # Apparently different versions of ruamel.yaml serialize this + # differently. + in ('Thing. Example value: "bar: baz"' 'Thing. Example value: "{bar: baz}"') + ) + + +def test_make_argument_description_with_array_adds_example(): + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': [1, '- foo', {'bar': 'baz'}], + }, + flag_name='flag', + ) + # Apparently different versions of ruamel.yaml serialize this + # differently. + in ( + 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + 'Thing. Example value: "[1, \'- foo\', {bar: baz}]"' + ) + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + arguments_group.add_argument( + '--foo[0].val', + action='store_true', + dest='--foo[0].val', + ) + + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo[25].val', + action='store_true', + default=False, + dest='foo[25].val', + required=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo.baz', + type=str, + metavar='BAZ', + help='help 2', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer', 'description': 'help 1'}, + 'baz': {'type': 'string', 'description': 'help 2'}, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo[0].bar', + type=int, + metavar='BAR', + help=object, + ).once() + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help 2', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'bar': { + 'type': 'integer', + 'description': 'help 1', + } + }, + }, + 'description': 'help 2', + } + }, + }, + unparsed_arguments=(), + ) + + def test_parse_arguments_with_no_arguments_uses_defaults(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments() + arguments = module.parse_arguments({}) global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -21,7 +154,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): def test_parse_arguments_with_multiple_config_flags_parses_as_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', '--config', 'otherconfig') + arguments = module.parse_arguments({}, '--config', 'myconfig', '--config', 'otherconfig') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig', 'otherconfig'] @@ -34,7 +167,7 @@ def test_parse_arguments_with_multiple_config_flags_parses_as_list(): def test_parse_arguments_with_action_after_config_path_omits_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'list', '--json') + arguments = module.parse_arguments({}, '--config', 'myconfig', 'list', '--json') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -45,7 +178,9 @@ def test_parse_arguments_with_action_after_config_path_omits_action(): def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey') + arguments = module.parse_arguments( + {}, '--config', 'myconfig', 'init', '--encryption', 'repokey' + ) global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -56,7 +191,7 @@ def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): def test_parse_arguments_with_action_and_positional_arguments_after_config_path_omits_action_and_arguments(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'borg', 'key', 'export') + arguments = module.parse_arguments({}, '--config', 'myconfig', 'borg', 'key', 'export') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -68,7 +203,7 @@ def test_parse_arguments_with_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--verbosity', '1') + arguments = module.parse_arguments({}, '--verbosity', '1') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -82,7 +217,7 @@ def test_parse_arguments_with_syslog_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--syslog-verbosity', '2') + arguments = module.parse_arguments({}, '--syslog-verbosity', '2') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -96,7 +231,7 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--log-file-verbosity', '-1') + arguments = module.parse_arguments({}, '--log-file-verbosity', '-1') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -109,7 +244,7 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default(): def test_parse_arguments_with_single_override_parses(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--override', 'foo.bar=baz') + arguments = module.parse_arguments({}, '--override', 'foo.bar=baz') global_arguments = arguments['global'] assert global_arguments.overrides == ['foo.bar=baz'] @@ -119,7 +254,7 @@ def test_parse_arguments_with_multiple_overrides_flags_parses(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) arguments = module.parse_arguments( - '--override', 'foo.bar=baz', '--override', 'foo.quux=7', '--override', 'this.that=8' + {}, '--override', 'foo.bar=baz', '--override', 'foo.quux=7', '--override', 'this.that=8' ) global_arguments = arguments['global'] @@ -127,7 +262,7 @@ def test_parse_arguments_with_multiple_overrides_flags_parses(): def test_parse_arguments_with_list_json_overrides_default(): - arguments = module.parse_arguments('list', '--json') + arguments = module.parse_arguments({}, 'list', '--json') assert 'list' in arguments assert arguments['list'].json is True @@ -136,7 +271,7 @@ def test_parse_arguments_with_list_json_overrides_default(): def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments() + arguments = module.parse_arguments({}) assert 'prune' in arguments assert 'create' in arguments @@ -146,14 +281,14 @@ def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): def test_parse_arguments_with_no_actions_passes_argument_to_relevant_actions(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--stats', '--list') + arguments = module.parse_arguments({}, '--stats', '--list') assert 'prune' in arguments - assert arguments['prune'].stats - assert arguments['prune'].list_archives + assert arguments['prune'].statistics + assert arguments['prune'].list_details assert 'create' in arguments - assert arguments['create'].stats - assert arguments['create'].list_files + assert arguments['create'].statistics + assert arguments['create'].list_details assert 'check' in arguments @@ -161,7 +296,7 @@ def test_parse_arguments_with_help_and_no_actions_shows_global_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('--help') + module.parse_arguments({}, '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -173,7 +308,7 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('create', '--help') + module.parse_arguments({}, 'create', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -185,7 +320,7 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys): def test_parse_arguments_with_action_before_global_options_parses_options(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('prune', '--verbosity', '2') + arguments = module.parse_arguments({}, 'prune', '--verbosity', '2') assert 'prune' in arguments assert arguments['global'].verbosity == 2 @@ -194,7 +329,7 @@ def test_parse_arguments_with_action_before_global_options_parses_options(): def test_parse_arguments_with_global_options_before_action_parses_options(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--verbosity', '2', 'prune') + arguments = module.parse_arguments({}, '--verbosity', '2', 'prune') assert 'prune' in arguments assert arguments['global'].verbosity == 2 @@ -203,7 +338,7 @@ def test_parse_arguments_with_global_options_before_action_parses_options(): def test_parse_arguments_with_prune_action_leaves_other_actions_disabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('prune') + arguments = module.parse_arguments({}, 'prune') assert 'prune' in arguments assert 'create' not in arguments @@ -213,7 +348,7 @@ def test_parse_arguments_with_prune_action_leaves_other_actions_disabled(): def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('create', 'check') + arguments = module.parse_arguments({}, 'create', 'check') assert 'prune' not in arguments assert 'create' in arguments @@ -224,60 +359,53 @@ def test_parse_arguments_disallows_invalid_argument(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--posix-me-harder') + module.parse_arguments({}, '--posix-me-harder') def test_parse_arguments_disallows_encryption_mode_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey') + module.parse_arguments({}, '--config', 'myconfig', '--encryption', 'repokey') def test_parse_arguments_allows_encryption_mode_with_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey') - - -def test_parse_arguments_requires_encryption_mode_with_init(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'init') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey') def test_parse_arguments_disallows_append_only_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--append-only') + module.parse_arguments({}, '--config', 'myconfig', '--append-only') def test_parse_arguments_disallows_storage_quota_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G') + module.parse_arguments({}, '--config', 'myconfig', '--storage-quota', '5G') def test_parse_arguments_allows_init_and_prune(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune') def test_parse_arguments_allows_init_and_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') def test_parse_arguments_allows_repository_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test' + {}, '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test' ) @@ -285,6 +413,7 @@ def test_parse_arguments_allows_repository_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( + {}, '--config', 'myconfig', 'mount', @@ -300,276 +429,247 @@ def test_parse_arguments_allows_repository_with_mount(): def test_parse_arguments_allows_repository_with_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'list', '--repository', 'test.borg') + module.parse_arguments({}, '--config', 'myconfig', 'list', '--repository', 'test.borg') def test_parse_arguments_disallows_archive_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', '--archive', 'test') def test_parse_arguments_disallows_paths_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--path', 'test') + module.parse_arguments({}, '--config', 'myconfig', '--path', 'test') def test_parse_arguments_disallows_other_actions_with_config_bootstrap(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('config', 'bootstrap', '--repository', 'test.borg', 'list') + module.parse_arguments({}, 'config', 'bootstrap', '--repository', 'test.borg', 'list') def test_parse_arguments_allows_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'extract', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'extract', '--archive', 'test') def test_parse_arguments_allows_archive_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - '--config', 'myconfig', 'mount', '--archive', 'test', '--mount-point', '/mnt' + {}, '--config', 'myconfig', 'mount', '--archive', 'test', '--mount-point', '/mnt' ) def test_parse_arguments_allows_archive_with_restore(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'restore', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'restore', '--archive', 'test') def test_parse_arguments_allows_archive_with_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'list', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'list', '--archive', 'test') def test_parse_arguments_requires_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'extract') + module.parse_arguments({}, '--config', 'myconfig', 'extract') def test_parse_arguments_requires_archive_with_restore(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'restore') + module.parse_arguments({}, '--config', 'myconfig', 'restore') def test_parse_arguments_requires_mount_point_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'mount', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'mount', '--archive', 'test') def test_parse_arguments_requires_mount_point_with_umount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'umount') + module.parse_arguments({}, '--config', 'myconfig', 'umount') def test_parse_arguments_allows_progress_before_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--progress', 'create', 'list') + module.parse_arguments({}, '--progress', 'create', 'list') def test_parse_arguments_allows_progress_after_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('create', '--progress', 'list') + module.parse_arguments({}, 'create', '--progress', 'list') def test_parse_arguments_allows_progress_and_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--progress', 'extract', '--archive', 'test', 'list') + module.parse_arguments({}, '--progress', 'extract', '--archive', 'test', 'list') def test_parse_arguments_disallows_progress_without_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--progress', 'list') + module.parse_arguments({}, '--progress', 'list') def test_parse_arguments_with_stats_and_create_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--stats', 'create', 'list') + module.parse_arguments({}, '--stats', 'create', 'list') def test_parse_arguments_with_stats_and_prune_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--stats', 'prune', 'list') + module.parse_arguments({}, '--stats', 'prune', 'list') def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--stats', 'list') + module.parse_arguments({}, '--stats', 'list') def test_parse_arguments_with_list_and_create_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--list', 'create') + module.parse_arguments({}, '--list', 'create') def test_parse_arguments_with_list_and_prune_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--list', 'prune') + module.parse_arguments({}, '--list', 'prune') def test_parse_arguments_with_list_flag_but_no_relevant_action_raises_value_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): - module.parse_arguments('--list', 'repo-create') - - -def test_parse_arguments_disallows_list_with_progress_for_create_action(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(ValueError): - module.parse_arguments('create', '--list', '--progress') - - -def test_parse_arguments_disallows_list_with_json_for_create_action(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments('create', '--list', '--json') + module.parse_arguments({}, '--list', 'repo-create') def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('list', '--json') - module.parse_arguments('info', '--json') + module.parse_arguments({}, 'list', '--json') + module.parse_arguments({}, 'info', '--json') def test_parse_arguments_disallows_json_with_both_list_and_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', 'info', '--json') + module.parse_arguments({}, 'list', 'info', '--json') def test_parse_arguments_disallows_json_with_both_list_and_repo_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', 'repo-info', '--json') + module.parse_arguments({}, 'list', 'repo-info', '--json') def test_parse_arguments_disallows_json_with_both_repo_info_and_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('repo-info', 'info', '--json') - - -def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments( - 'transfer', - '--source-repository', - 'source.borg', - '--archive', - 'foo', - '--match-archives', - 'sh:*bar', - ) + module.parse_arguments({}, 'repo-info', 'info', '--json') def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'list', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_repo_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('repo-list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'repo-list', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_info_with_both_archive_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--archive', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'info', '--archive', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_info_with_both_archive_and_prefix(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--archive', 'foo', '--prefix', 'bar') + module.parse_arguments({}, 'info', '--archive', 'foo', '--prefix', 'bar') def test_parse_arguments_disallows_info_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'info', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_check_only_extract_does_not_raise_extract_subparser_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('check', '--only', 'extract') + module.parse_arguments({}, 'check', '--only', 'extract') def test_parse_arguments_extract_archive_check_does_not_raise_check_subparser_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('extract', '--archive', 'check') + module.parse_arguments({}, 'extract', '--archive', 'check') def test_parse_arguments_extract_with_check_only_extract_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract') + module.parse_arguments({}, 'extract', '--archive', 'name', 'check', '--only', 'extract') def test_parse_arguments_bootstrap_without_config_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('bootstrap') + module.parse_arguments({}, 'bootstrap') def test_parse_arguments_config_with_no_subaction_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('config') + module.parse_arguments({}, 'config') def test_parse_arguments_config_with_help_shows_config_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', '--help') + module.parse_arguments({}, 'config', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -582,7 +682,7 @@ def test_parse_arguments_config_with_subaction_but_missing_flags_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', 'bootstrap') + module.parse_arguments({}, 'config', 'bootstrap') assert exit.value.code == 2 @@ -591,7 +691,7 @@ def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(cap flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', 'bootstrap', '--help') + module.parse_arguments({}, 'config', 'bootstrap', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -601,26 +701,30 @@ def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(cap def test_parse_arguments_config_with_subaction_and_required_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg') + module.parse_arguments({}, 'config', 'bootstrap', '--repository', 'repo.borg') def test_parse_arguments_config_with_subaction_and_global_flags_at_start_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg') + module.parse_arguments( + {}, '--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg' + ) def test_parse_arguments_config_with_subaction_and_global_flags_at_end_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1') + module.parse_arguments( + {}, 'config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1' + ) def test_parse_arguments_config_with_subaction_and_explicit_config_file_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - 'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml' + {}, 'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml' ) @@ -628,10 +732,23 @@ def test_parse_arguments_with_borg_action_and_dry_run_raises(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--dry-run', 'borg', 'list') + module.parse_arguments({}, '--dry-run', 'borg', 'list') def test_parse_arguments_with_borg_action_and_no_dry_run_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('borg', 'list') + module.parse_arguments({}, 'borg', 'list') + + +def test_parse_arguments_with_argument_from_schema_does_not_raise(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + module.parse_arguments( + { + 'type': 'object', + 'properties': {'foo': {'type': 'object', 'properties': {'bar': {'type': 'integer'}}}}, + }, + '--foo.bar', + '3', + ) diff --git a/tests/integration/config/test_arguments.py b/tests/integration/config/test_arguments.py new file mode 100644 index 000000000..9bcc37ff9 --- /dev/null +++ b/tests/integration/config/test_arguments.py @@ -0,0 +1,34 @@ +import pytest + +from borgmatic.config import arguments as module + + +def test_convert_value_type_passes_through_non_string_value(): + assert module.convert_value_type([1, 2], 'array') == [1, 2] + + +def test_convert_value_type_passes_through_string_option_type(): + assert module.convert_value_type('foo', 'string') == 'foo' + + +def test_convert_value_type_parses_array_option_type(): + assert module.convert_value_type('[foo, bar]', 'array') == ['foo', 'bar'] + + +def test_convert_value_type_with_array_option_type_and_no_array_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar}', 'array') + + +def test_convert_value_type_parses_object_option_type(): + assert module.convert_value_type('{foo: bar}', 'object') == {'foo': 'bar'} + + +def test_convert_value_type_with_invalid_value_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar', 'object') + + +def test_convert_value_type_with_unknown_option_type_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar}', 'thingy') diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 426224038..71c0abc56 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -21,9 +21,9 @@ def test_schema_to_sample_configuration_comments_out_non_default_options(): 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('source_directories', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('source_directories', {'type': 'string', 'example': 'Example 3'}), ] ), } @@ -47,9 +47,9 @@ def test_schema_to_sample_configuration_comments_out_non_source_config_options() 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), } @@ -76,9 +76,9 @@ def test_schema_to_sample_configuration_comments_out_non_default_options_in_sequ 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('source_directories', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('source_directories', {'type': 'string', 'example': 'Example 3'}), ] ), }, @@ -105,9 +105,9 @@ def test_schema_to_sample_configuration_comments_out_non_source_config_options_i 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), }, diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 9cd5c9802..e74989c69 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -58,7 +58,9 @@ def test_parse_configuration_transforms_file_into_mapping(): ''' ) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': ['/home', '/etc'], @@ -86,7 +88,9 @@ def test_parse_configuration_passes_through_quoted_punctuation(): ''' ) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': [f'/home/{string.punctuation}'], @@ -119,7 +123,9 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): ''', ) - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_inlines_include_inside_deprecated_section(): @@ -145,7 +151,9 @@ def test_parse_configuration_inlines_include_inside_deprecated_section(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': ['/home'], @@ -181,7 +189,9 @@ def test_parse_configuration_merges_include(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': ['/home'], @@ -196,7 +206,9 @@ def test_parse_configuration_merges_include(): def test_parse_configuration_raises_for_missing_config_file(): with pytest.raises(FileNotFoundError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_raises_for_missing_schema_file(): @@ -208,14 +220,18 @@ def test_parse_configuration_raises_for_missing_schema_file(): builtins.should_receive('open').with_args('/tmp/schema.yaml').and_raise(FileNotFoundError) with pytest.raises(FileNotFoundError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_raises_for_syntax_error(): mock_config_and_schema('foo:\nbar') with pytest.raises(ValueError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_raises_for_validation_error(): @@ -228,7 +244,9 @@ def test_parse_configuration_raises_for_validation_error(): ) with pytest.raises(module.Validation_error): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_applies_overrides(): @@ -245,7 +263,10 @@ def test_parse_configuration_applies_overrides(): ) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', overrides=['local_path=borg2'] + '/tmp/config.yaml', + '/tmp/schema.yaml', + arguments={'global': flexmock()}, + overrides=['local_path=borg2'], ) assert config == { @@ -273,7 +294,9 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in ) flexmock(os).should_receive('getenv').replace_with(lambda variable_name, default: default) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': ['/home'], diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index 7a21e7152..ece911977 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -105,7 +105,7 @@ def test_get_config_paths_translates_ssh_command_argument_to_config(): flexmock(module.borgmatic.config.paths).should_receive( 'get_borgmatic_source_directory' ).and_return('/source') - config = flexmock() + config = {} flexmock(module).should_receive('make_bootstrap_config').and_return(config) bootstrap_arguments = flexmock( repository='repo', @@ -267,11 +267,11 @@ def test_run_bootstrap_does_not_raise(): archive='archive', destination='dest', strip_components=1, - progress=False, user_runtime_directory='/borgmatic', ssh_command=None, local_path='borg7', remote_path='borg8', + progress=None, ) global_arguments = flexmock( dry_run=False, @@ -299,7 +299,7 @@ def test_run_bootstrap_does_not_raise(): def test_run_bootstrap_translates_ssh_command_argument_to_config(): - config = flexmock() + config = {} flexmock(module).should_receive('make_bootstrap_config').and_return(config) flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml']) bootstrap_arguments = flexmock( @@ -307,11 +307,11 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): archive='archive', destination='dest', strip_components=1, - progress=False, user_runtime_directory='/borgmatic', ssh_command='ssh -i key', local_path='borg7', remote_path='borg8', + progress=None, ) global_arguments = flexmock( dry_run=False, @@ -333,13 +333,12 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): 'repo', 'archive', object, - config, + {'progress': False}, object, object, extract_to_stdout=False, destination_path='dest', strip_components=1, - progress=False, local_path='borg7', remote_path='borg8', ).and_return(extract_process).once() diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 1ee77393b..e7eee5c4c 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -577,7 +577,6 @@ def test_collect_spot_check_source_paths_parses_borg_output(): borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -625,7 +624,6 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false() borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=False, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -673,7 +671,6 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -721,7 +718,6 @@ def test_collect_spot_check_source_paths_skips_directories(): borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -860,14 +856,13 @@ def test_collect_spot_check_source_paths_uses_working_directory(): flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args( dry_run=True, repository_path='repo', - config=object, + config={'working_directory': '/working/dir', 'list_details': True}, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version=object, global_arguments=object, borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 14dcfaadc..04a3a25e4 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -9,7 +9,10 @@ def test_compact_actions_calls_hooks_for_configured_repository(): flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() compact_arguments = flexmock( - repository=None, progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=None, + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -34,7 +37,10 @@ def test_compact_runs_with_selected_repository(): flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() compact_arguments = flexmock( - repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -59,7 +65,10 @@ def test_compact_bails_if_repository_does_not_match(): ).once().and_return(False) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').never() compact_arguments = flexmock( - repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 1f6ca7dce..6761f8f46 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -443,9 +443,9 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): create_arguments = flexmock( repository=None, progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -484,9 +484,9 @@ def test_run_create_runs_with_selected_repository(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -516,9 +516,9 @@ def test_run_create_bails_if_repository_does_not_match(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -538,6 +538,72 @@ def test_run_create_bails_if_repository_does_not_match(): ) +def test_run_create_with_both_list_and_json_errors(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never() + flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + statistics=flexmock(), + json=True, + list_details=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'list_details': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + +def test_run_create_with_both_list_and_progress_errors(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never() + flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + statistics=flexmock(), + json=False, + list_details=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'list_details': True, 'progress': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + def test_run_create_produces_json(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( @@ -561,9 +627,9 @@ def test_run_create_produces_json(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=True, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index aea54af34..d32aa6fe8 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -13,7 +13,7 @@ def test_run_export_tar_does_not_raise(): paths=flexmock(), destination=flexmock(), tar_filter=flexmock(), - list_files=flexmock(), + list_details=flexmock(), strip_components=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -27,3 +27,81 @@ def test_run_export_tar_does_not_raise(): local_path=None, remote_path=None, ) + + +def test_run_export_tar_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.export_tar).should_receive('export_tar_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + tar_filter=object, + strip_components=object, + ).once() + export_tar_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + paths=flexmock(), + destination=flexmock(), + tar_filter=flexmock(), + list_details=False, + strip_components=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_export_tar( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version=None, + export_tar_arguments=export_tar_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_export_tar_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.export_tar).should_receive('export_tar_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + tar_filter=object, + strip_components=object, + ).once() + export_tar_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + paths=flexmock(), + destination=flexmock(), + tar_filter=flexmock(), + list_details=None, + strip_components=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_export_tar( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version=None, + export_tar_arguments=export_tar_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index c483adcc5..1504d4cec 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -27,3 +27,79 @@ def test_run_extract_calls_hooks(): local_path=None, remote_path=None, ) + + +def test_run_extract_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + destination_path=object, + strip_components=object, + ).once() + extract_arguments = flexmock( + paths=flexmock(), + progress=False, + destination=flexmock(), + strip_components=flexmock(), + archive=flexmock(), + repository='repo', + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_extract( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'repositories': ['repo'], 'progress': True}, + local_borg_version=None, + extract_arguments=extract_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_extract_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + destination_path=object, + strip_components=object, + ).once() + extract_arguments = flexmock( + paths=flexmock(), + progress=None, + destination=flexmock(), + strip_components=flexmock(), + archive=flexmock(), + repository='repo', + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_extract( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'repositories': ['repo'], 'progress': True}, + local_borg_version=None, + extract_arguments=extract_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index 762a83c20..b37e50fb5 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -7,7 +7,7 @@ def test_run_prune_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() - prune_arguments = flexmock(repository=None, stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock(repository=None, statistics=flexmock(), list_details=flexmock()) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( @@ -29,7 +29,9 @@ def test_run_prune_runs_with_selected_repository(): 'repositories_match' ).once().and_return(True) flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() - prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock( + repository=flexmock(), statistics=flexmock(), list_details=flexmock() + ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( @@ -51,7 +53,9 @@ def test_run_prune_bails_if_repository_does_not_match(): 'repositories_match' ).once().and_return(False) flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').never() - prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock( + repository=flexmock(), statistics=flexmock(), list_details=flexmock() + ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( diff --git a/tests/unit/actions/test_repo_create.py b/tests/unit/actions/test_repo_create.py index 8bb350b17..0b54818e4 100644 --- a/tests/unit/actions/test_repo_create.py +++ b/tests/unit/actions/test_repo_create.py @@ -1,9 +1,10 @@ +import pytest from flexmock import flexmock from borgmatic.actions import repo_create as module -def test_run_repo_create_does_not_raise(): +def test_run_repo_create_with_encryption_mode_argument_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') @@ -14,7 +15,7 @@ def test_run_repo_create_does_not_raise(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -28,6 +29,57 @@ def test_run_repo_create_does_not_raise(): ) +def test_run_repo_create_with_encryption_mode_option_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') + arguments = flexmock( + encryption_mode=None, + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_directories=flexmock(), + ) + + module.run_repo_create( + repository={'path': 'repo', 'encryption': flexmock()}, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + +def test_run_repo_create_without_encryption_mode_raises(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') + arguments = flexmock( + encryption_mode=None, + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_directories=flexmock(), + ) + + with pytest.raises(ValueError): + module.run_repo_create( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + def test_run_repo_create_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( @@ -41,7 +93,7 @@ def test_run_repo_create_bails_if_repository_does_not_match(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -53,3 +105,91 @@ def test_run_repo_create_bails_if_repository_does_not_match(): local_path=None, remote_path=None, ) + + +def test_run_repo_create_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + append_only=False, + storage_quota=0, + make_parent_directories=False, + local_path=object, + remote_path=object, + ).once() + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=False, + storage_quota=0, + make_parent_directories=False, + ) + + module.run_repo_create( + repository={ + 'path': 'repo', + 'append_only': True, + 'storage_quota': '10G', + 'make_parent_directories': True, + }, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + +def test_run_repo_create_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + append_only=True, + storage_quota='10G', + make_parent_directories=True, + local_path=object, + remote_path=object, + ).once() + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=None, + storage_quota=None, + make_parent_directories=None, + ) + + module.run_repo_create( + repository={ + 'path': 'repo', + 'append_only': True, + 'storage_quota': '10G', + 'make_parent_directories': True, + }, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py index 03d259bec..be4eda257 100644 --- a/tests/unit/actions/test_transfer.py +++ b/tests/unit/actions/test_transfer.py @@ -1,3 +1,4 @@ +import pytest from flexmock import flexmock from borgmatic.actions import transfer as module @@ -6,7 +7,7 @@ from borgmatic.actions import transfer as module def test_run_transfer_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.transfer).should_receive('transfer_archives') - transfer_arguments = flexmock() + transfer_arguments = flexmock(archive=None) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_transfer( @@ -18,3 +19,21 @@ def test_run_transfer_does_not_raise(): local_path=None, remote_path=None, ) + + +def test_run_transfer_with_archive_and_match_archives_raises(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.transfer).should_receive('transfer_archives') + transfer_arguments = flexmock(archive='foo') + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + module.run_transfer( + repository={'path': 'repo'}, + config={'match_archives': 'foo*'}, + local_borg_version=None, + transfer_arguments=transfer_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index cb3ce67cd..0a1f5361d 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -155,22 +155,6 @@ def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_arc assert flags == ('--match-archives', 'sh:foo-*') -def test_make_archive_filter_flags_prefers_check_arguments_match_archives_to_config_match_archives(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'baz-*', None, '1.2.3' - ).and_return(('--match-archives', 'sh:baz-*')) - - flags = module.make_archive_filter_flags( - '1.2.3', - {'match_archives': 'bar-{now}', 'prefix': ''}, # noqa: FS003 - ('archives',), - check_arguments=flexmock(match_archives='baz-*'), - ) - - assert flags == ('--match-archives', 'sh:baz-*') - - def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( @@ -332,7 +316,7 @@ def test_get_repository_id_with_missing_json_keys_raises(): def test_check_archives_with_progress_passes_through_to_borg(): - config = {} + config = {'progress': True} flexmock(module).should_receive('make_check_name_flags').with_args( {'repository'}, () ).and_return(()) @@ -353,7 +337,7 @@ def test_check_archives_with_progress_passes_through_to_borg(): config=config, local_borg_version='1.2.3', check_arguments=flexmock( - progress=True, + progress=None, repair=None, only_checks=None, force=None, diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 882dac691..7fff77286 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -27,7 +27,7 @@ def insert_execute_command_mock( COMPACT_COMMAND = ('borg', 'compact') -def test_compact_segments_calls_borg_with_parameters(): +def test_compact_segments_calls_borg_with_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) @@ -40,7 +40,7 @@ def test_compact_segments_calls_borg_with_parameters(): ) -def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): +def test_compact_segments_with_log_info_calls_borg_with_info_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) @@ -54,7 +54,7 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): ) -def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): +def test_compact_segments_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) @@ -110,7 +110,7 @@ def test_compact_segments_with_exit_codes_calls_borg_using_them(): ) -def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_compact_segments_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) @@ -124,21 +124,20 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter ) -def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): +def test_compact_segments_with_progress_calls_borg_with_progress_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--progress', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, ) -def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_parameter(): +def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--cleanup-commits', 'repo'), logging.INFO) @@ -152,21 +151,20 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p ) -def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): +def test_compact_segments_with_threshold_calls_borg_with_threshold_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--threshold', '20', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - config={}, + config={'compact_threshold': 20}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - threshold=20, ) -def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): +def test_compact_segments_with_umask_calls_borg_with_umask_flags(): config = {'umask': '077'} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.INFO) @@ -180,7 +178,7 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): ) -def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): +def test_compact_segments_with_log_json_calls_borg_with_log_json_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--log-json', 'repo'), logging.INFO) @@ -193,7 +191,7 @@ def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): ) -def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_flags(): config = {'lock_wait': 5} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index ad7c044ea..80b80ee18 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -631,12 +631,12 @@ def test_make_base_create_command_includes_list_flags_in_borg_command(): config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], + 'list_details': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/run/borgmatic', - list_files=True, ) assert create_flags == ('borg', 'create', '--list', '--filter', 'FOO') @@ -962,7 +962,7 @@ def test_make_base_create_command_with_non_existent_directory_and_source_directo ) -def test_create_archive_calls_borg_with_parameters(): +def test_create_archive_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1029,7 +1029,7 @@ def test_create_archive_calls_borg_with_environment(): ) -def test_create_archive_with_log_info_calls_borg_with_info_parameter(): +def test_create_archive_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1096,7 +1096,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): ) -def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): +def test_create_archive_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1196,7 +1196,6 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats(): local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - stats=True, ) @@ -1271,7 +1270,7 @@ def test_create_archive_with_exit_codes_calls_borg_using_them(): ) -def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_output_log_level(): +def test_create_archive_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1296,12 +1295,12 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'statistics': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - stats=True, ) @@ -1334,16 +1333,16 @@ def test_create_archive_with_files_calls_borg_with_answer_output_log_level(): 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'list_details': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - list_files=True, ) -def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list(): +def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_flag_and_no_list(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1369,16 +1368,16 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, ) -def test_create_archive_with_progress_calls_borg_with_progress_parameter(): +def test_create_archive_with_progress_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1403,16 +1402,16 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, ) -def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter(): +def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER processes = flexmock() @@ -1459,12 +1458,12 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, stream_processes=processes, ) @@ -1532,7 +1531,6 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag(): global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', json=True, - stats=True, ) assert json_output == '[]' diff --git a/tests/unit/borg/test_delete.py b/tests/unit/borg/test_delete.py index 6d8614b66..770f411db 100644 --- a/tests/unit/borg/test_delete.py +++ b/tests/unit/borg/test_delete.py @@ -21,7 +21,7 @@ def test_make_delete_command_includes_log_info(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -43,7 +43,7 @@ def test_make_delete_command_includes_log_debug(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -67,7 +67,7 @@ def test_make_delete_command_includes_dry_run(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=True, log_json=False), local_path='borg', remote_path=None, @@ -91,7 +91,7 @@ def test_make_delete_command_includes_remote_path(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path='borg1', @@ -114,7 +114,7 @@ def test_make_delete_command_includes_umask(): repository={'path': 'repo'}, config={'umask': '077'}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -138,7 +138,7 @@ def test_make_delete_command_includes_log_json(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', remote_path=None, @@ -162,7 +162,7 @@ def test_make_delete_command_includes_lock_wait(): repository={'path': 'repo'}, config={'lock_wait': 5}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -171,7 +171,7 @@ def test_make_delete_command_includes_lock_wait(): assert command == ('borg', 'delete', '--lock-wait', '5', 'repo') -def test_make_delete_command_includes_list(): +def test_make_delete_command_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( 'list', True @@ -184,9 +184,9 @@ def test_make_delete_command_includes_list(): command = module.make_delete_command( repository={'path': 'repo'}, - config={}, + config={'list_details': True}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=True, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=None, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -207,7 +207,7 @@ def test_make_delete_command_includes_force(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=1, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=1, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -228,7 +228,7 @@ def test_make_delete_command_includes_force_twice(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=2, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=2, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -252,7 +252,7 @@ def test_make_delete_command_includes_archive(): config={}, local_borg_version='1.2.3', delete_arguments=flexmock( - list_archives=False, force=0, match_archives=None, archive='archive' + list_details=False, force=0, match_archives=None, archive='archive' ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -277,7 +277,7 @@ def test_make_delete_command_includes_match_archives(): config={}, local_borg_version='1.2.3', delete_arguments=flexmock( - list_archives=False, force=0, match_archives='sh:foo*', archive='archive' + list_details=False, force=0, match_archives='sh:foo*', archive='archive' ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -287,8 +287,12 @@ def test_make_delete_command_includes_match_archives(): assert command == ('borg', 'delete', '--match-archives', 'sh:foo*', 'repo') +LOGGING_ANSWER = flexmock() + + def test_delete_archives_with_archive_calls_borg_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -308,6 +312,7 @@ def test_delete_archives_with_archive_calls_borg_delete(): def test_delete_archives_with_match_archives_calls_borg_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -328,6 +333,7 @@ def test_delete_archives_with_match_archives_calls_borg_delete(): @pytest.mark.parametrize('argument_name', module.ARCHIVE_RELATED_ARGUMENT_NAMES[2:]) def test_delete_archives_with_archive_related_argument_calls_borg_delete(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -347,6 +353,7 @@ def test_delete_archives_with_archive_related_argument_calls_borg_delete(argumen def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').once() flexmock(module).should_receive('make_delete_command').never() @@ -359,7 +366,7 @@ def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete config={}, local_borg_version=flexmock(), delete_arguments=flexmock( - list_archives=True, force=False, cache_only=False, keep_security_info=False + list_details=True, force=False, cache_only=False, keep_security_info=False ), global_arguments=flexmock(), ) @@ -367,6 +374,7 @@ def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete def test_delete_archives_calls_borg_delete_with_working_directory(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() command = flexmock() flexmock(module).should_receive('make_delete_command').and_return(command) diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index f45026f35..3fb50129b 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -144,7 +144,7 @@ def test_export_tar_archive_calls_borg_with_umask_flags(): ) -def test_export_tar_archive_calls_borg_with_log_json_parameter(): +def test_export_tar_archive_calls_borg_with_log_json_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -186,7 +186,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_flags(): ) -def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): +def test_export_tar_archive_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -230,7 +230,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_flags(): ) -def test_export_tar_archive_calls_borg_with_dry_run_parameter(): +def test_export_tar_archive_calls_borg_with_dry_run_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -273,7 +273,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_flags(): ) -def test_export_tar_archive_calls_borg_with_list_parameter(): +def test_export_tar_archive_calls_borg_with_list_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -290,14 +290,13 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): archive='archive', paths=None, destination_path='test.tar', - config={}, + config={'list_details': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - list_files=True, ) -def test_export_tar_archive_calls_borg_with_strip_components_parameter(): +def test_export_tar_archive_calls_borg_with_strip_components_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -320,7 +319,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): ) -def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): +def test_export_tar_archive_skips_abspath_for_remote_repository_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index de392f246..a11155f87 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -580,7 +580,7 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): ) -def test_extract_archive_calls_borg_with_progress_parameter(): +def test_extract_archive_calls_borg_with_progress_flag(): flexmock(module.os.path).should_receive('abspath').and_return('repo') flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) @@ -606,10 +606,9 @@ def test_extract_archive_calls_borg_with_progress_parameter(): repository='repo', archive='archive', paths=None, - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, ) @@ -622,10 +621,9 @@ def test_extract_archive_with_progress_and_extract_to_stdout_raises(): repository='repo', archive='archive', paths=None, - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, extract_to_stdout=True, ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index 5866b4abf..23f416bb7 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -380,7 +380,7 @@ def test_make_info_command_with_match_archives_flag_passes_through_to_command(): command = module.make_info_command( repository_path='repo', - config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}', 'match_archives': 'sh:foo-*'}, # noqa: FS003 local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 3f2da33a1..10e8ebf98 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -135,32 +135,6 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): assert result == expected -def test_make_prune_flags_without_prefix_uses_match_archives_flag_instead_of_option(): - config = { - 'archive_name_format': 'bar-{now}', # noqa: FS003 - 'match_archives': 'foo*', - 'keep_daily': 1, - 'prefix': None, - } - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'baz*', 'bar-{now}', '1.2.3' # noqa: FS003 - ).and_return(('--match-archives', 'sh:bar-*')).once() - - result = module.make_prune_flags( - config, flexmock(match_archives='baz*'), local_borg_version='1.2.3' - ) - - expected = ( - '--keep-daily', - '1', - '--match-archives', - 'sh:bar-*', # noqa: FS003 - ) - - assert result == expected - - def test_make_prune_flags_without_prefix_uses_match_archives_option(): config = { 'archive_name_format': 'bar-{now}', # noqa: FS003 @@ -215,7 +189,7 @@ def test_prune_archives_calls_borg_with_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -237,7 +211,7 @@ def test_prune_archives_with_log_info_calls_borg_with_info_flag(): insert_execute_command_mock(PRUNE_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -259,7 +233,7 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_flag(): insert_execute_command_mock(PRUNE_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -280,7 +254,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_flag(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -301,7 +275,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): ).and_return(False) insert_execute_command_mock(('borg1',) + PRUNE_COMMAND[1:] + ('repo',), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -328,7 +302,7 @@ def test_prune_archives_with_exit_codes_calls_borg_using_them(): borg_exit_codes=borg_exit_codes, ) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -349,7 +323,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -361,7 +335,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): ) -def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): +def test_prune_archives_with_stats_config_calls_borg_with_stats_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -371,18 +345,18 @@ def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_ ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) - prune_arguments = flexmock(stats=True, list_archives=False) + prune_arguments = flexmock(statistics=None, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'statistics': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_log_level(): +def test_prune_archives_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -392,11 +366,11 @@ def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_l ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) - prune_arguments = flexmock(stats=False, list_archives=True) + prune_arguments = flexmock(statistics=False, list_details=None) module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'list_details': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -414,7 +388,7 @@ def test_prune_archives_with_umask_calls_borg_with_umask_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -435,7 +409,7 @@ def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--log-json', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -457,7 +431,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -478,7 +452,7 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -546,7 +520,7 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag ) prune_arguments = flexmock( - stats=False, list_archives=False, newer='1d', newest='1y', older='1m', oldest='1w' + statistics=False, list_details=False, newer='1d', newest='1y', older='1m', oldest='1w' ) module.prune_archives( dry_run=False, @@ -570,7 +544,7 @@ def test_prune_archives_calls_borg_with_working_directory(): PRUNE_COMMAND + ('repo',), logging.INFO, working_directory='/working/dir' ) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -581,7 +555,7 @@ def test_prune_archives_calls_borg_with_working_directory(): ) -def test_prune_archives_calls_borg_with_flags_and_when_feature_available(): +def test_prune_archives_calls_borg_without_stats_when_feature_is_not_available(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -591,11 +565,11 @@ def test_prune_archives_calls_borg_with_flags_and_when_feature_available(): ).and_return(True) insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.ANSWER) - prune_arguments = flexmock(stats=True, list_archives=False) + prune_arguments = flexmock(statistics=True, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'statistics': True}, local_borg_version='2.0.0b10', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index d65a86ed7..d0d527bf7 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -267,7 +267,7 @@ def test_recreate_with_log_json(): ) -def test_recreate_with_list_filter_flags(): +def test_recreate_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) @@ -288,10 +288,10 @@ def test_recreate_with_list_filter_flags(): module.recreate_archive( repository='repo', archive='archive', - config={}, + config={'list_details': True}, local_borg_version='1.2.3', recreate_arguments=flexmock( - list=True, + list=None, target=None, comment=None, timestamp=None, diff --git a/tests/unit/borg/test_repo_create.py b/tests/unit/borg/test_repo_create.py index dc645f12d..9e3a674d0 100644 --- a/tests/unit/borg/test_repo_create.py +++ b/tests/unit/borg/test_repo_create.py @@ -228,7 +228,29 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'append_only': True}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), + encryption_mode='repokey', + append_only=True, + ) + + +def test_create_repository_with_append_only_config_calls_borg_with_append_only_flag(): + insert_repo_info_command_not_found_mock() + insert_repo_create_command_mock(REPO_CREATE_COMMAND + ('--append-only', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + config={'append_only': True}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -252,7 +274,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'storage_quota': '5G'}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -274,11 +296,11 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'make_parent_directories': True}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', - make_parent_dirs=True, + make_parent_directories=True, ) diff --git a/tests/unit/borg/test_repo_delete.py b/tests/unit/borg/test_repo_delete.py index 66e737782..4af66085c 100644 --- a/tests/unit/borg/test_repo_delete.py +++ b/tests/unit/borg/test_repo_delete.py @@ -19,7 +19,7 @@ def test_make_repo_delete_command_with_feature_available_runs_borg_repo_delete() repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -40,7 +40,7 @@ def test_make_repo_delete_command_without_feature_available_runs_borg_delete(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -62,7 +62,7 @@ def test_make_repo_delete_command_includes_log_info(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -84,7 +84,7 @@ def test_make_repo_delete_command_includes_log_debug(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -108,7 +108,7 @@ def test_make_repo_delete_command_includes_dry_run(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=True, log_json=False), local_path='borg', remote_path=None, @@ -132,7 +132,7 @@ def test_make_repo_delete_command_includes_remote_path(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path='borg1', @@ -155,7 +155,7 @@ def test_make_repo_delete_command_includes_umask(): repository={'path': 'repo'}, config={'umask': '077'}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -179,7 +179,7 @@ def test_make_repo_delete_command_includes_log_json(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', remote_path=None, @@ -203,7 +203,7 @@ def test_make_repo_delete_command_includes_lock_wait(): repository={'path': 'repo'}, config={'lock_wait': 5}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -225,9 +225,9 @@ def test_make_repo_delete_command_includes_list(): command = module.make_repo_delete_command( repository={'path': 'repo'}, - config={}, + config={'list_details': True}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=True, force=0), + repo_delete_arguments=flexmock(list_details=True, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -248,7 +248,7 @@ def test_make_repo_delete_command_includes_force(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=1), + repo_delete_arguments=flexmock(list_details=False, force=1), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -269,7 +269,7 @@ def test_make_repo_delete_command_includes_force_twice(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=2), + repo_delete_arguments=flexmock(list_details=False, force=2), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, diff --git a/tests/unit/borg/test_repo_list.py b/tests/unit/borg/test_repo_list.py index da750d7ad..73f567f58 100644 --- a/tests/unit/borg/test_repo_list.py +++ b/tests/unit/borg/test_repo_list.py @@ -664,7 +664,7 @@ def test_make_repo_list_command_with_match_archives_calls_borg_with_match_archiv command = module.make_repo_list_command( repository_path='repo', - config={}, + config={'match_archives': 'foo-*'}, local_borg_version='1.2.3', repo_list_arguments=flexmock( archive=None, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index edd2131b0..b2b9886d6 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -193,7 +193,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, repository_path='repo', - config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}', 'match_archives': 'sh:foo*'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None @@ -436,12 +436,15 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ) -def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): +def test_transfer_archives_with_progress_calls_borg_with_progress_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('progress', True).and_return( + ('--progress',) + ) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) @@ -458,10 +461,10 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - config={}, + config={'progress': True}, local_borg_version='2.3.4', transfer_arguments=flexmock( - archive=None, progress=True, match_archives=None, source_repository=None + archive=None, progress=None, match_archives=None, source_repository=None ), global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/commands/completion/test_flag.py b/tests/unit/commands/completion/test_flag.py new file mode 100644 index 000000000..06e999792 --- /dev/null +++ b/tests/unit/commands/completion/test_flag.py @@ -0,0 +1,20 @@ +from borgmatic.commands.completion import flag as module + + +def test_variants_passes_through_non_list_index_flag_name(): + assert tuple(module.variants('foo')) == ('foo',) + + +def test_variants_broadcasts_list_index_flag_name_with_a_range_of_indices(): + assert tuple(module.variants('foo[0].bar')) == ( + 'foo[0].bar', + 'foo[1].bar', + 'foo[2].bar', + 'foo[3].bar', + 'foo[4].bar', + 'foo[5].bar', + 'foo[6].bar', + 'foo[7].bar', + 'foo[8].bar', + 'foo[9].bar', + ) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index d10cf887d..f30942772 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -575,3 +575,755 @@ def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): with pytest.raises(ValueError): module.parse_arguments_for_actions(('config',), action_parsers, global_parser) + + +def test_make_argument_description_with_object_adds_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('{foo: example}') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + 'example': {'foo': 'example'}, + }, + flag_name='flag', + ) + == 'Thing. Example value: "{foo: example}"' + ) + + +def test_make_argument_description_without_description_and_with_object_sets_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('{foo: example}') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'type': 'object', + 'example': {'foo': 'example'}, + }, + flag_name='flag', + ) + == 'Example value: "{foo: example}"' + ) + + +def test_make_argument_description_with_object_skips_missing_example(): + flexmock(module.ruamel.yaml).should_receive('YAML').never() + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + }, + flag_name='flag', + ) + == 'Thing.' + ) + + +def test_make_argument_description_with_array_adds_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('[example]') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) + == 'Thing. Example value: "[example]"' + ) + + +def test_make_argument_description_without_description_and_with_array_sets_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('[example]') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) + == 'Example value: "[example]"' + ) + + +def test_make_argument_description_with_array_skips_missing_example(): + flexmock(module.ruamel.yaml).should_receive('YAML').never() + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + }, + flag_name='flag', + ) + == 'Thing.' + ) + + +def test_make_argument_description_with_array_index_in_flag_name_adds_to_description(): + assert 'list element' in module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'something', + }, + flag_name='flag[0]', + ) + + +def test_make_argument_description_without_description_and_with_array_index_in_flag_name_sets_description(): + assert 'list element' in module.make_argument_description( + schema={ + 'type': 'something', + }, + flag_name='flag[0]', + ) + + +def test_make_argument_description_escapes_percent_character(): + assert ( + module.make_argument_description( + schema={ + 'description': '% Thing.', + 'type': 'something', + }, + flag_name='flag', + ) + == '%% Thing.' + ) + + +def test_add_array_element_arguments_without_array_index_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo', + ) + + +def test_add_array_element_arguments_with_help_flag_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--help', '--bar'), + flag_name='foo[0]', + ) + + +def test_add_array_element_arguments_without_any_flags_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo[0]', + ) + + +# Use this instead of a flexmock because it's not easy to check the type() of a flexmock instance. +Group_action = collections.namedtuple( + 'Group_action', + ( + 'option_strings', + 'choices', + 'default', + 'nargs', + 'required', + 'type', + ), + defaults=( + flexmock(), + flexmock(), + flexmock(), + flexmock(), + flexmock(), + ), +) + + +def test_add_array_element_arguments_without_array_index_flags_bails(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--bar'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_with_non_matching_array_index_flags_bails(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--bar[25].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_with_identical_array_index_flag_bails(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[0].val', 'fooval', '--bar'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_without_action_type_in_registry_bails(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': bool}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val', + action='store_stuff', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_equals_sign(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val', + action='store_stuff', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val=fooval', '--bar[1].val=barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_dashes(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val-and-stuff',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val-and-stuff', + action='store_stuff', + choices=object, + default=object, + dest='foo[25].val_and_stuff', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val-and-stuff', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val-and-stuff', + ) + + +def test_add_arguments_from_schema_with_non_dict_schema_bails(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').never() + flexmock(module.borgmatic.config.schema).should_receive('parse_type').never() + arguments_group.should_receive('add_argument').never() + + module.add_arguments_from_schema( + arguments_group=arguments_group, schema='foo', unparsed_arguments=() + ) + + +def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( + 'help 2' + ) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return( + int + ).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo.baz', + type=str, + metavar='BAZ', + help='help 2', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer'}, + 'baz': {'type': 'str'}, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_uses_first_non_null_type_from_multi_type_object(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': ['null', 'object', 'boolean'], + 'properties': { + 'bar': {'type': 'integer'}, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_empty_multi_type_raises(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int) + arguments_group.should_receive('add_argument').never() + flexmock(module).should_receive('add_array_element_arguments').never() + + with pytest.raises(ValueError): + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': [], + 'properties': { + 'bar': {'type': 'integer'}, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_propertyless_option_adds_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_of_scalars_adds_multiple_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').with_args( + 'integer', object=str, array=str + ).and_return(int) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').with_args( + 'array', object=str, array=str + ).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo[0]', + type=int, + metavar='FOO[0]', + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'integer', + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_of_objects_adds_multiple_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( + 'help 2' + ) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return( + int + ).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo[0].bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help 2', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + flexmock(module).should_receive('add_array_element_arguments').with_args( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo[0].bar', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'bar': { + 'type': 'integer', + } + }, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_boolean_adds_two_valueless_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--no-foo', + dest='foo', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_nested_boolean_adds_two_valueless_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar.baz-quux', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo.bar.no-baz-quux', + dest='foo.bar.baz_quux', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'baz_quux': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + names=('foo', 'bar'), + ) + + +def test_add_arguments_from_schema_with_boolean_with_name_prefixed_with_no_adds_two_valueless_flags_and_removes_the_no_for_one(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--no-foo', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + dest='no_foo', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'no_foo': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_skips_omitted_flag_name(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--match-archives', + type=object, + metavar=object, + help=object, + ).never() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'match_archives': { + 'type': 'string', + }, + 'foo': { + 'type': 'string', + }, + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_rewrites_option_name_to_flag_name(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo-and-stuff', + type=str, + metavar='FOO_AND_STUFF', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo_and_stuff': { + 'type': 'string', + }, + }, + }, + unparsed_arguments=(), + ) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 3abdff875..57f3cf678 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1578,6 +1578,7 @@ def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env configs, config_paths, logs = tuple( module.load_configurations( ('test.yaml', 'other.yaml'), + arguments=flexmock(), resolve_env=resolve_env, ) ) @@ -1590,7 +1591,9 @@ def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env def test_load_configurations_logs_warning_for_permission_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(PermissionError) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple( + module.load_configurations(('test.yaml',), arguments=flexmock()) + ) assert configs == {} assert config_paths == [] @@ -1600,7 +1603,9 @@ def test_load_configurations_logs_warning_for_permission_error(): def test_load_configurations_logs_critical_for_parse_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple( + module.load_configurations(('test.yaml',), arguments=flexmock()) + ) assert configs == {} assert config_paths == [] diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py new file mode 100644 index 000000000..fdf5f0f58 --- /dev/null +++ b/tests/unit/config/test_arguments.py @@ -0,0 +1,234 @@ +import pytest +from flexmock import flexmock + +from borgmatic.config import arguments as module + + +def test_set_values_without_keys_bails(): + config = {'option': 'value'} + module.set_values(config=config, keys=(), value=5) + + assert config == {'option': 'value'} + + +def test_set_values_with_keys_adds_them_to_config(): + config = {'option': 'value'} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'option': 'value', 'foo': {'bar': {'baz': 5}}} + + +def test_set_values_with_one_existing_key_adds_others_to_config(): + config = {'foo': {'other': 'value'}} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'foo': {'other': 'value', 'bar': {'baz': 5}}} + + +def test_set_values_with_two_existing_keys_adds_others_to_config(): + config = {'foo': {'bar': {'other': 'value'}}} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'foo': {'bar': {'other': 'value', 'baz': 5}}} + + +def test_set_values_with_list_index_key_adds_it_to_config(): + config = {'foo': {'bar': [{'option': 'value'}, {'other': 'thing'}]}} + + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + assert config == {'foo': {'bar': [{'option': 'value'}, {'other': 'thing', 'baz': 5}]}} + + +def test_set_values_with_list_index_key_out_of_range_raises(): + config = {'foo': {'bar': [{'option': 'value'}]}} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + +def test_set_values_with_final_list_index_key_out_of_range_raises(): + config = {'foo': {'bar': [{'option': 'value'}]}} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]'), value=5) + + +def test_set_values_with_list_index_key_missing_list_and_out_of_range_raises(): + config = {'other': 'value'} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + +def test_set_values_with_final_list_index_key_adds_it_to_config(): + config = {'foo': {'bar': [1, 2]}} + + module.set_values(config=config, keys=('foo', 'bar[1]'), value=5) + + assert config == {'foo': {'bar': [1, 5]}} + + +def test_type_for_option_with_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={'type': 'object', 'properties': {'foo': {'type': 'integer'}}}, + option_keys=('foo',), + ) + == 'integer' + ) + + +def test_type_for_option_with_nested_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': {'type': 'object', 'properties': {'bar': {'type': 'boolean'}}} + }, + }, + option_keys=('foo', 'bar'), + ) + == 'boolean' + ) + + +def test_type_for_option_with_missing_nested_option_finds_nothing(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': {'type': 'object', 'properties': {'other': {'type': 'integer'}}} + }, + }, + option_keys=('foo', 'bar'), + ) + is None + ) + + +def test_type_for_option_with_typeless_nested_option_finds_nothing(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': {'foo': {'type': 'object', 'properties': {'bar': {'example': 5}}}}, + }, + option_keys=('foo', 'bar'), + ) + is None + ) + + +def test_type_for_option_with_list_index_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': {'foo': {'type': 'array', 'items': {'type': 'integer'}}}, + }, + option_keys=('foo[0]',), + ) + == 'integer' + ) + + +def test_type_for_option_with_nested_list_index_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': {'type': 'object', 'properties': {'bar': {'type': 'integer'}}}, + } + }, + }, + option_keys=('foo[0]', 'bar'), + ) + == 'integer' + ) + + +def test_prepare_arguments_for_config_converts_arguments_to_keys(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ( + (('my_option', 'sub_option'), 'value1'), + (('other_option',), 'value2'), + ) + + +def test_prepare_arguments_for_config_skips_option_with_none_value(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': None, 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ((('other_option',), 'value2'),) + + +def test_prepare_arguments_for_config_skips_option_missing_from_schema(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object'}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ((('other_option',), 'value2'),) + + +def test_apply_arguments_to_config_does_not_raise(): + flexmock(module).should_receive('prepare_arguments_for_config').and_return( + ( + (('foo', 'bar'), 'baz'), + (('one', 'two'), 'three'), + ) + ) + flexmock(module).should_receive('set_values') + + module.apply_arguments_to_config(config={}, schema={}, arguments={'global': flexmock()}) diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 27dcd01f7..ff71b9fb6 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -4,94 +4,27 @@ from flexmock import flexmock from borgmatic.config import generate as module -def test_get_properties_with_simple_object(): - schema = { - 'type': 'object', - 'properties': dict( - [ - ('field1', {'example': 'Example'}), - ] - ), - } - - assert module.get_properties(schema) == schema['properties'] - - -def test_get_properties_merges_oneof_list_properties(): - schema = { - 'type': 'object', - 'oneOf': [ - { - 'properties': dict( - [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ] - ), - }, - { - 'properties': dict( - [ - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - ], - } - - assert module.get_properties(schema) == dict( - schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] - ) - - -def test_get_properties_interleaves_oneof_list_properties(): - schema = { - 'type': 'object', - 'oneOf': [ - { - 'properties': dict( - [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - { - 'properties': dict( - [ - ('field4', {'example': 'Example 4'}), - ('field5', {'example': 'Example 5'}), - ] - ), - }, - ], - } - - assert module.get_properties(schema) == dict( - [ - ('field1', {'example': 'Example 1'}), - ('field4', {'example': 'Example 4'}), - ('field2', {'example': 'Example 2'}), - ('field5', {'example': 'Example 5'}), - ('field3', {'example': 'Example 3'}), - ] - ) - - def test_schema_to_sample_configuration_generates_config_map_with_examples(): schema = { 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), } - flexmock(module).should_receive('get_properties').and_return(schema['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) flexmock(module).should_receive('add_comments_to_configuration_object') @@ -106,6 +39,35 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): ) +def test_schema_to_sample_configuration_with_empty_object_generates_config_map_with_example(): + schema = { + 'type': 'object', + 'example': { + 'foo': 'Example 1', + 'baz': 'Example 2', + }, + } + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return({}) + flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) + flexmock(module).should_receive('add_comments_to_configuration_object') + + config = module.schema_to_sample_configuration(schema) + + assert config == dict( + [ + ('foo', 'Example 1'), + ('baz', 'Example 2'), + ] + ) + + def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example(): flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') @@ -122,11 +84,26 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e 'items': { 'type': 'object', 'properties': dict( - [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ] ), }, } - flexmock(module).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'array', {'array'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['items']['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') @@ -142,11 +119,26 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_m 'items': { 'type': ['object', 'null'], 'properties': dict( - [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ] ), }, } - flexmock(module).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'array', {'array'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + ['object', 'null'], {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['items']['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index abd7e54dc..29fe25e80 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -359,6 +359,11 @@ def test_normalize_commands_moves_individual_command_hooks_to_unified_commands( {'repositories': [{'path': '/repo', 'label': 'foo'}]}, False, ), + ( + {'repositories': [{'path': None, 'label': 'foo'}]}, + {'repositories': []}, + False, + ), ( {'prefix': 'foo'}, {'prefix': 'foo'}, diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py new file mode 100644 index 000000000..8af890fb3 --- /dev/null +++ b/tests/unit/config/test_schema.py @@ -0,0 +1,160 @@ +import pytest + +from borgmatic.config import schema as module + + +def test_get_properties_with_simple_object(): + schema = { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'example': 'Example'}), + ] + ), + } + + assert module.get_properties(schema) == schema['properties'] + + +def test_get_properties_merges_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] + ) + + +def test_get_properties_interleaves_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field4', {'example': 'Example 4'}), + ('field5', {'example': 'Example 5'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + [ + ('field1', {'example': 'Example 1'}), + ('field4', {'example': 'Example 4'}), + ('field2', {'example': 'Example 2'}), + ('field5', {'example': 'Example 5'}), + ('field3', {'example': 'Example 3'}), + ] + ) + + +def test_parse_type_maps_schema_type_to_python_type(): + module.parse_type('boolean') == bool + + +def test_parse_type_with_unknown_schema_type_raises(): + with pytest.raises(ValueError): + module.parse_type('what') + + +def test_parse_type_respect_overrides_when_mapping_types(): + module.parse_type('boolean', boolean=int) == int + + +@pytest.mark.parametrize( + 'schema_type,target_types,match,expected_result', + ( + ( + 'string', + {'integer', 'string', 'boolean'}, + None, + True, + ), + ( + 'string', + {'integer', 'boolean'}, + None, + False, + ), + ( + 'string', + {'integer', 'string', 'boolean'}, + all, + True, + ), + ( + 'string', + {'integer', 'boolean'}, + all, + False, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean'}, + None, + True, + ), + ( + ['string', 'array'], + {'integer', 'boolean'}, + None, + False, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean', 'array'}, + all, + True, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean'}, + all, + False, + ), + ( + ['string', 'array'], + {'integer', 'boolean'}, + all, + False, + ), + ), +) +def test_compare_types_returns_whether_schema_type_matches_target_types( + schema_type, target_types, match, expected_result +): + if match: + assert module.compare_types(schema_type, target_types, match) == expected_result + else: + assert module.compare_types(schema_type, target_types) == expected_result diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index dffaafc0d..d179735f4 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -44,19 +44,23 @@ def test_interactive_console_true_when_isatty_and_TERM_is_not_dumb(capsys): assert module.interactive_console() is True -def test_should_do_markup_respects_no_color_value(): - flexmock(module.os.environ).should_receive('get').and_return(None) +def test_should_do_markup_respects_json_enabled_value(): + flexmock(module.os.environ).should_receive('get').never() flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=True) is False def test_should_do_markup_respects_config_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': False}}, json_enabled=False) is False + ) flexmock(module).should_receive('interactive_console').and_return(True).once() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': True}}) is True + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': True}}, json_enabled=False) is True + ) def test_should_do_markup_prefers_any_false_config_value(): @@ -65,11 +69,11 @@ def test_should_do_markup_prefers_any_false_config_value(): assert ( module.should_do_markup( - no_color=False, configs={ 'foo.yaml': {'color': True}, 'bar.yaml': {'color': False}, }, + json_enabled=False, ) is False ) @@ -83,14 +87,16 @@ def test_should_do_markup_respects_PY_COLORS_environment_variable(): flexmock(module).should_receive('to_bool').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True -def test_should_do_markup_prefers_no_color_value_to_config_value(): +def test_should_do_markup_prefers_json_enabled_value_to_config_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={'foo.yaml': {'color': True}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': True}}, json_enabled=True) is False + ) def test_should_do_markup_prefers_config_value_to_environment_variables(): @@ -98,7 +104,9 @@ def test_should_do_markup_prefers_config_value_to_environment_variables(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': False}}, json_enabled=False) is False + ) def test_should_do_markup_prefers_no_color_value_to_environment_variables(): @@ -106,14 +114,14 @@ def test_should_do_markup_prefers_no_color_value_to_environment_variables(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_respects_interactive_console_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value(): @@ -124,7 +132,7 @@ def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value(): @@ -132,7 +140,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value(): flexmock(module.os.environ).should_receive('get').with_args('NO_COLOR', None).and_return('True') flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_respects_NO_COLOR_environment_variable(): @@ -140,7 +148,7 @@ def test_should_do_markup_respects_NO_COLOR_environment_variable(): flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_ignores_empty_NO_COLOR_environment_variable(): @@ -148,7 +156,7 @@ def test_should_do_markup_ignores_empty_NO_COLOR_environment_variable(): flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None) flexmock(module).should_receive('interactive_console').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_NO_COLOR_to_PY_COLORS(): @@ -160,7 +168,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_PY_COLORS(): ) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_multi_stream_handler_logs_to_handler_for_log_level():