diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py new file mode 100644 index 00000000..9254da6d --- /dev/null +++ b/borgmatic/actions/config/bootstrap.py @@ -0,0 +1,84 @@ +import json +import logging +import os + +import borgmatic.borg.extract +import borgmatic.borg.rlist +import borgmatic.config.validate +import borgmatic.hooks.command +from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY + +logger = logging.getLogger(__name__) + + +def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): + ''' + Given: + The bootstrap arguments, which include the repository and archive name, borgmatic source directory, + destination directory, and whether to strip components. + The global arguments, which include the dry run flag + and the local borg version, + Return: + The config paths from the manifest.json file in the borgmatic source directory after extracting it from the + repository. + ''' + borgmatic_source_directory = ( + bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY + ) + borgmatic_manifest_path = os.path.expanduser( + os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json') + ) + extract_process = borgmatic.borg.extract.extract_archive( + global_arguments.dry_run, + bootstrap_arguments.repository, + borgmatic.borg.rlist.resolve_archive_name( + bootstrap_arguments.repository, + bootstrap_arguments.archive, + {}, + local_borg_version, + global_arguments, + ), + [borgmatic_manifest_path], + {}, + {}, + local_borg_version, + global_arguments, + extract_to_stdout=True, + ) + + manifest_data = json.loads(extract_process.stdout.read()) + + return manifest_data['config_paths'] + + +def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): + ''' + Run the "bootstrap" action for the given repository. + ''' + manifest_config_paths = get_config_paths( + bootstrap_arguments, global_arguments, local_borg_version + ) + + for config_path in manifest_config_paths: + logger.info('Bootstrapping config path %s', config_path) + + borgmatic.borg.extract.extract_archive( + global_arguments.dry_run, + bootstrap_arguments.repository, + borgmatic.borg.rlist.resolve_archive_name( + bootstrap_arguments.repository, + bootstrap_arguments.archive, + {}, + local_borg_version, + global_arguments, + ), + [config_path], + {}, + {}, + local_borg_version, + global_arguments, + extract_to_stdout=False, + destination_path=bootstrap_arguments.destination, + strip_components=bootstrap_arguments.strip_components, + progress=bootstrap_arguments.progress, + ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index cb8b1cf4..4d83634b 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -1,15 +1,51 @@ import json import logging +import os + +try: + import importlib_metadata +except ModuleNotFoundError: # pragma: nocover + import importlib.metadata as importlib_metadata import borgmatic.borg.create import borgmatic.config.validate import borgmatic.hooks.command import borgmatic.hooks.dispatch import borgmatic.hooks.dump +from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY logger = logging.getLogger(__name__) +def create_borgmatic_manifest(location, config_paths, dry_run): + ''' + Create a borgmatic manifest file to store the paths to the configuration files used to create + the archive. + ''' + if dry_run: + return + + borgmatic_source_directory = location.get( + 'borgmatic_source_directory', DEFAULT_BORGMATIC_SOURCE_DIRECTORY + ) + + borgmatic_manifest_path = os.path.expanduser( + os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json') + ) + + if not os.path.exists(borgmatic_manifest_path): + os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True) + + with open(borgmatic_manifest_path, 'w') as config_list_file: + json.dump( + { + 'borgmatic_version': importlib_metadata.version('borgmatic'), + 'config_paths': config_paths, + }, + config_list_file, + ) + + def run_create( config_filename, repository, @@ -59,6 +95,9 @@ def run_create( location, global_arguments.dry_run, ) + create_borgmatic_manifest( + location, global_arguments.used_config_paths, global_arguments.dry_run + ) stream_processes = [process for processes in active_dumps.values() for process in processes] json_output = borgmatic.borg.create.create_archive( diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index e3b70eb5..618376a5 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -351,7 +351,9 @@ def create_archive( sources = deduplicate_directories( map_directories_to_devices( expand_directories( - tuple(location_config.get('source_directories', ())) + borgmatic_source_directories + tuple(location_config.get('source_directories', ())) + + borgmatic_source_directories + + tuple(global_arguments.used_config_paths) ) ), additional_directory_devices=map_directories_to_devices( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6039e4fa..64858521 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,4 +1,6 @@ +import argparse import collections +import itertools from argparse import Action, ArgumentParser from borgmatic.config import collect @@ -9,6 +11,8 @@ SUBPARSER_ALIASES = { 'compact': [], 'create': ['-C'], 'check': ['-k'], + 'config': [], + 'config_bootstrap': [], 'extract': ['-x'], 'export-tar': [], 'mount': ['-m'], @@ -24,6 +28,27 @@ SUBPARSER_ALIASES = { } +def get_unparsable_arguments(remaining_subparser_arguments): + ''' + Determine the remaining arguments that no subparsers have consumed. + ''' + if remaining_subparser_arguments: + remaining_arguments = [ + argument + for argument in dict.fromkeys( + itertools.chain.from_iterable(remaining_subparser_arguments) + ).keys() + if all( + argument in subparser_arguments + for subparser_arguments in remaining_subparser_arguments + ) + ] + else: + remaining_arguments = [] + + return remaining_arguments + + def parse_subparser_arguments(unparsed_arguments, subparsers): ''' Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser @@ -40,6 +65,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): for subparser_name, aliases in SUBPARSER_ALIASES.items() for alias in aliases } + subcommand_parsers_mapping = { + 'config': ['bootstrap'], + } # If the "borg" action is used, skip all other subparsers. This avoids confusion like # "borg list" triggering borgmatic's own list action. @@ -56,7 +84,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): # If a parsed value happens to be the same as the name of a subparser, remove it from the # remaining arguments. This prevents, for instance, "check --only extract" from triggering # the "extract" subparser. - parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments) + parsed, unused_remaining = subparser.parse_known_args( + [argument for argument in unparsed_arguments if argument != canonical_name] + ) for value in vars(parsed).values(): if isinstance(value, str): if value in subparsers: @@ -66,7 +96,16 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): if item in subparsers: remaining_arguments.remove(item) - arguments[canonical_name] = parsed + arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed + + for argument in arguments: + if not arguments[argument]: + if not any( + subcommand in arguments for subcommand in subcommand_parsers_mapping[argument] + ): + raise ValueError( + f'Missing subcommand for {argument}. Expected one of {subcommand_parsers_mapping[argument]}' + ) # If no actions are explicitly requested, assume defaults. if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments: @@ -77,13 +116,22 @@ def parse_subparser_arguments(unparsed_arguments, subparsers): remaining_arguments = list(unparsed_arguments) - # Now ask each subparser, one by one, to greedily consume arguments. - for subparser_name, subparser in subparsers.items(): + # Now ask each subparser, one by one, to greedily consume arguments, from last to first. This + # allows subparsers to consume arguments before their parent subparsers do. + remaining_subparser_arguments = [] + + for subparser_name, subparser in reversed(subparsers.items()): if subparser_name not in arguments.keys(): continue subparser = subparsers[subparser_name] - unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments) + unused_parsed, remaining = subparser.parse_known_args( + [argument for argument in unparsed_arguments if argument != subparser_name] + ) + remaining_subparser_arguments.append(remaining) + + if remaining_subparser_arguments: + remaining_arguments = get_unparsable_arguments(remaining_subparser_arguments) # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the # "borg" action. @@ -109,7 +157,7 @@ class Extend_action(Action): items = getattr(namespace, self.dest, None) if items: - items.extend(values) + items.extend(values) # pragma: no cover else: setattr(namespace, self.dest, list(values)) @@ -563,6 +611,71 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + config_parser = subparsers.add_parser( + 'config', + aliases=SUBPARSER_ALIASES['config'], + help='Perform configuration file related operations', + description='Perform configuration file related operations', + add_help=False, + ) + + config_group = config_parser.add_argument_group('config arguments') + config_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') + + config_subparsers = config_parser.add_subparsers( + title='config subcommands', + description='Valid subcommands for config', + help='Additional help', + ) + + config_bootstrap_parser = config_subparsers.add_parser( + 'bootstrap', + aliases=SUBPARSER_ALIASES['config_bootstrap'], + help='Extract the config files used to create a borgmatic repository', + description='Extract config files that were used to create a borgmatic repository during the "create" operation', + add_help=False, + ) + config_bootstrap_group = config_bootstrap_parser.add_argument_group( + 'config bootstrap arguments' + ) + config_bootstrap_group.add_argument( + '--repository', + help='Path of repository to extract config files from', + required=True, + ) + config_bootstrap_group.add_argument( + '--borgmatic-source-directory', + help='Path that stores the config files used to create an archive and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic', + ) + config_bootstrap_group.add_argument( + '--archive', + help='Name of archive to extract config files from, defaults to "latest"', + default='latest', + ) + config_bootstrap_group.add_argument( + '--destination', + metavar='PATH', + dest='destination', + help='Directory to extract config files into, defaults to /', + default='/', + ) + config_bootstrap_group.add_argument( + '--strip-components', + type=lambda number: number if number == 'all' else int(number), + metavar='NUMBER', + help='Number of leading path components to remove from each extracted path or "all" to strip all leading path components. Skip paths with fewer elements', + ) + config_bootstrap_group.add_argument( + '--progress', + dest='progress', + default=False, + action='store_true', + help='Display progress for each file as it is extracted', + ) + config_bootstrap_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) + export_tar_parser = subparsers.add_parser( 'export-tar', aliases=SUBPARSER_ALIASES['export-tar'], @@ -973,7 +1086,28 @@ def make_parsers(): ) borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - return top_level_parser, subparsers + merged_subparsers = argparse._SubParsersAction( + None, None, metavar=None, dest='merged', parser_class=None + ) + + merged_subparsers = merge_subparsers(subparsers, config_subparsers) + + return top_level_parser, merged_subparsers + + +def merge_subparsers(*subparsers): + ''' + Merge multiple subparsers into a single subparser. + ''' + merged_subparsers = argparse._SubParsersAction( + None, None, metavar=None, dest='merged', parser_class=None + ) + + for subparser in subparsers: + for name, subparser in subparser.choices.items(): + merged_subparsers._name_parser_map[name] = subparser + + return merged_subparsers def parse_arguments(*unparsed_arguments): @@ -986,6 +1120,16 @@ def parse_arguments(*unparsed_arguments): arguments, remaining_arguments = parse_subparser_arguments( unparsed_arguments, subparsers.choices ) + + if ( + 'bootstrap' in arguments.keys() + and 'config' in arguments.keys() + and len(arguments.keys()) > 2 + ): + raise ValueError( + 'The bootstrap action cannot be combined with other actions. Please run it separately.' + ) + arguments['global'] = top_level_parser.parse_args(remaining_arguments) if arguments['global'].excludes_filename: diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 535c048e..69b7e8dd 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -18,6 +18,7 @@ import borgmatic.actions.borg import borgmatic.actions.break_lock import borgmatic.actions.check import borgmatic.actions.compact +import borgmatic.actions.config.bootstrap import borgmatic.actions.create import borgmatic.actions.export_tar import borgmatic.actions.extract @@ -622,11 +623,37 @@ def collect_configuration_run_summary_logs(configs, arguments): if 'extract' in arguments or 'mount' in arguments: validate.guard_single_repository_selected(repository, configs) - validate.guard_configuration_contains_repository(repository, configs) + if 'bootstrap' not in arguments: + validate.guard_configuration_contains_repository(repository, configs) except ValueError as error: yield from log_error_records(str(error)) return + if 'bootstrap' in arguments: + # no configuration file is needed for bootstrap + local_borg_version = borg_version.local_borg_version({}, 'borg') + try: + borgmatic.actions.config.bootstrap.run_bootstrap( + arguments['bootstrap'], arguments['global'], local_borg_version + ) + yield logging.makeLogRecord( + dict( + levelno=logging.INFO, + levelname='INFO', + msg='Bootstrap successful', + ) + ) + except ( + CalledProcessError, + ValueError, + OSError, + json.JSONDecodeError, + KeyError, + ) as error: + yield from log_error_records('Error running bootstrap', error) + + return + if not configs: yield from log_error_records( f"{' '.join(arguments['global'].config_paths)}: No valid configuration files found", @@ -733,6 +760,7 @@ def main(): # pragma: no cover sys.exit(0) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) + global_arguments.used_config_paths = list(config_filenames) configs, parse_logs = load_configurations( config_filenames, global_arguments.overrides, global_arguments.resolve_env ) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 4990fc4f..60fe884c 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -1,3 +1,5 @@ +import argparse + import pytest from flexmock import flexmock @@ -298,6 +300,13 @@ def test_parse_arguments_disallows_paths_unless_action_consumes_it(): module.parse_arguments('--config', 'myconfig', '--path', 'test') +def test_parse_arguments_disallows_other_actions_with_config_bootstrap(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('config', 'bootstrap', '--repository', 'test.borg', 'list') + + def test_parse_arguments_allows_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) @@ -523,3 +532,26 @@ def test_parse_arguments_extract_with_check_only_extract_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract') + + +def test_merging_two_subparser_collections_merges_their_choices(): + top_level_parser = argparse.ArgumentParser() + + subparsers = top_level_parser.add_subparsers() + subparser1 = subparsers.add_parser('subparser1') + + subparser2 = subparsers.add_parser('subparser2') + subsubparsers = subparser2.add_subparsers() + subsubparser1 = subsubparsers.add_parser('subsubparser1') + + merged_subparsers = argparse._SubParsersAction( + None, None, metavar=None, dest='merged', parser_class=None + ) + + merged_subparsers = module.merge_subparsers(subparsers, subsubparsers) + + assert merged_subparsers.choices == { + 'subparser1': subparser1, + 'subparser2': subparser2, + 'subsubparser1': subsubparser1, + } diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py new file mode 100644 index 00000000..d1b21511 --- /dev/null +++ b/tests/unit/actions/config/test_bootstrap.py @@ -0,0 +1,56 @@ +from flexmock import flexmock + +from borgmatic.actions.config import bootstrap as module + + +def test_get_config_paths_returns_list_of_config_paths(): + bootstrap_arguments = flexmock( + borgmatic_source_directory=None, + repository='repo', + archive='archive', + ) + global_arguments = flexmock( + dry_run=False, + ) + local_borg_version = flexmock() + extract_process = flexmock( + stdout=flexmock( + read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}', + ), + ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + extract_process + ) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + 'archive' + ) + assert module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) == [ + '/borgmatic/config.yaml' + ] + + +def test_run_bootstrap_does_not_raise(): + bootstrap_arguments = flexmock( + repository='repo', + archive='archive', + destination='dest', + strip_components=1, + progress=False, + borgmatic_source_directory='/borgmatic', + ) + global_arguments = flexmock( + dry_run=False, + ) + local_borg_version = flexmock() + extract_process = flexmock( + stdout=flexmock( + read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}', + ), + ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + extract_process + ).twice() + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + 'archive' + ) + module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 2b724085..5846b8ad 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -1,3 +1,5 @@ +import sys + from flexmock import flexmock from borgmatic.actions import create as module @@ -7,6 +9,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() + flexmock(module).should_receive('create_borgmatic_manifest').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( @@ -19,7 +22,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): json=flexmock(), list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) list( module.run_create( @@ -45,6 +48,7 @@ def test_run_create_runs_with_selected_repository(): 'repositories_match' ).once().and_return(True) flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() + flexmock(module).should_receive('create_borgmatic_manifest').once() create_arguments = flexmock( repository=flexmock(), progress=flexmock(), @@ -52,7 +56,7 @@ def test_run_create_runs_with_selected_repository(): json=flexmock(), list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) list( module.run_create( @@ -78,6 +82,7 @@ def test_run_create_bails_if_repository_does_not_match(): 'repositories_match' ).once().and_return(False) flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + flexmock(module).should_receive('create_borgmatic_manifest').never() create_arguments = flexmock( repository=flexmock(), progress=flexmock(), @@ -85,7 +90,7 @@ def test_run_create_bails_if_repository_does_not_match(): json=flexmock(), list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) list( module.run_create( @@ -103,3 +108,42 @@ def test_run_create_bails_if_repository_does_not_match(): remote_path=None, ) ) + + +def test_create_borgmatic_manifest_creates_manifest_file(): + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.os).should_receive('makedirs').and_return(True) + + flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0') + flexmock(module.json).should_receive('dump').and_return(True) + + module.create_borgmatic_manifest({}, 'test.yaml', False) + + +def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_source_directory(): + flexmock(module.os.path).should_receive('join').with_args( + '/borgmatic', 'bootstrap', 'manifest.json' + ).and_return('/borgmatic/bootstrap/manifest.json') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.os).should_receive('makedirs').and_return(True) + + flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0') + flexmock(sys.modules['builtins']).should_receive('open').with_args( + '/borgmatic/bootstrap/manifest.json', 'w' + ).and_return( + flexmock( + __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None), + __exit__=lambda *args: None, + ) + ) + flexmock(module.json).should_receive('dump').and_return(True) + + module.create_borgmatic_manifest( + {'borgmatic_source_directory': '/borgmatic'}, 'test.yaml', False + ) + + +def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run(): + flexmock(module.os.path).should_receive('expanduser').never() + + module.create_borgmatic_manifest({}, 'test.yaml', True) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index e0462e2d..c798c446 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -492,7 +492,7 @@ def test_create_archive_calls_borg_with_parameters(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -536,7 +536,7 @@ def test_create_archive_calls_borg_with_environment(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -582,7 +582,56 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), + ) + + +def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sources_and_config_paths(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('deduplicate_directories').and_return( + ('foo', 'bar', '/etc/borgmatic/config.yaml') + ) + flexmock(module).should_receive('map_directories_to_devices').and_return({}) + flexmock(module).should_receive('expand_directories').with_args([]).and_return(()) + flexmock(module).should_receive('expand_directories').with_args( + ('foo', 'bar', '/etc/borgmatic/config.yaml') + ).and_return(('foo', 'bar', '/etc/borgmatic/config.yaml')) + flexmock(module).should_receive('expand_directories').with_args([]).and_return(()) + flexmock(module).should_receive('pattern_root_directories').and_return([]) + flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) + flexmock(module).should_receive('expand_home_directories').and_return(()) + flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module).should_receive('ensure_files_readable') + flexmock(module).should_receive('make_pattern_flags').and_return(()) + flexmock(module).should_receive('make_exclude_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',) + ) + environment = {'BORG_THINGY': 'YUP'} + flexmock(module.environment).should_receive('make_environment').and_return(environment) + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('/etc/borgmatic/config.yaml',), + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=environment, + ) + + module.create_archive( + dry_run=False, + repository_path='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + }, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']), ) @@ -628,7 +677,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -672,7 +721,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -713,7 +762,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, ) @@ -758,7 +807,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -799,7 +848,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, ) @@ -843,7 +892,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -889,7 +938,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stats=True, ) @@ -933,7 +982,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte }, storage_config={'checkpoint_interval': 600}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -976,7 +1025,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume }, storage_config={'checkpoint_volume': 1024}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1019,7 +1068,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param }, storage_config={'chunker_params': '1,2,3,4'}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1062,7 +1111,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( }, storage_config={'compression': 'rle'}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1111,7 +1160,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ }, storage_config={'upload_rate_limit': 100}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1157,7 +1206,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1201,7 +1250,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1251,7 +1300,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1305,7 +1354,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1361,7 +1410,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1416,7 +1465,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1471,7 +1520,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1515,7 +1564,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1558,7 +1607,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), local_path='borg1', ) @@ -1602,7 +1651,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), remote_path='borg1', ) @@ -1646,7 +1695,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): }, storage_config={'umask': 740}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1689,7 +1738,7 @@ def test_create_archive_with_log_json_calls_borg_with_log_json_parameters(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=True), + global_arguments=flexmock(log_json=True, used_config_paths=[]), ) @@ -1732,7 +1781,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): }, storage_config={'lock_wait': 5}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -1775,7 +1824,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stats=True, ) @@ -1819,7 +1868,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), list_files=True, ) @@ -1869,7 +1918,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), progress=True, ) @@ -1913,7 +1962,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), progress=True, ) @@ -1974,7 +2023,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), progress=True, stream_processes=processes, ) @@ -2039,7 +2088,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, ) @@ -2107,7 +2156,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, ) @@ -2172,7 +2221,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, ) @@ -2213,7 +2262,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, ) @@ -2256,7 +2305,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), json=True, stats=True, ) @@ -2304,7 +2353,7 @@ def test_create_archive_with_source_directories_glob_expands(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2348,7 +2397,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2391,7 +2440,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2434,7 +2483,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): }, storage_config={'archive_name_format': 'ARCHIVE_NAME'}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2478,7 +2527,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2522,7 +2571,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2565,7 +2614,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): }, storage_config={'extra_borg_options': {'create': '--extra --options'}}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) @@ -2626,7 +2675,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), stream_processes=processes, ) @@ -2652,7 +2701,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ }, storage_config={}, local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(log_json=False, used_config_paths=[]), ) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 9354cf5e..ef8fd1ea 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -1,5 +1,6 @@ import collections +import pytest from flexmock import flexmock from borgmatic.commands import arguments as module @@ -164,3 +165,45 @@ def test_parse_subparser_arguments_parses_borg_options_and_skips_other_subparser assert arguments == {'borg': action_namespace} assert arguments['borg'].options == ['list'] assert remaining_arguments == [] + + +def test_parse_subparser_arguments_raises_error_when_no_subparser_is_specified(): + action_namespace = flexmock(options=[]) + subparsers = { + 'config': flexmock(parse_known_args=lambda arguments: (action_namespace, ['config'])), + } + + with pytest.raises(ValueError): + module.parse_subparser_arguments(('config',), subparsers) + + +@pytest.mark.parametrize( + 'arguments, expected', + [ + ( + ( + ('--latest', 'archive', 'prune', 'extract', 'list', '--test-flag'), + ('--latest', 'archive', 'check', 'extract', 'list', '--test-flag'), + ('prune', 'check', 'list', '--test-flag'), + ('prune', 'check', 'extract', '--test-flag'), + ), + [ + '--test-flag', + ], + ), + ( + ( + ('--latest', 'archive', 'prune', 'extract', 'list'), + ('--latest', 'archive', 'check', 'extract', 'list'), + ('prune', 'check', 'list'), + ('prune', 'check', 'extract'), + ), + [], + ), + ((), []), + ], +) +def test_get_unparsable_arguments_returns_remaining_arguments_that_no_subparser_can_parse( + arguments, expected +): + assert module.get_unparsable_arguments(arguments) == expected diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 4ca802d5..60fb7568 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1000,6 +1000,41 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): assert {log.levelno for log in logs} == {logging.INFO} +def test_collect_configuration_run_summary_logs_info_for_success_with_bootstrap(): + flexmock(module.validate).should_receive('guard_single_repository_selected').never() + flexmock(module.validate).should_receive('guard_configuration_contains_repository').never() + flexmock(module).should_receive('run_configuration').never() + flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap') + arguments = { + 'bootstrap': flexmock(repository='repo'), + 'global': flexmock(monitoring_verbosity=1, dry_run=False), + } + + logs = tuple( + module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + ) + assert {log.levelno for log in logs} == {logging.INFO} + + +def test_collect_configuration_run_summary_logs_error_on_bootstrap_failure(): + flexmock(module.validate).should_receive('guard_single_repository_selected').never() + flexmock(module.validate).should_receive('guard_configuration_contains_repository').never() + flexmock(module).should_receive('run_configuration').never() + flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').and_raise( + ValueError + ) + arguments = { + 'bootstrap': flexmock(repository='repo'), + 'global': flexmock(monitoring_verbosity=1, dry_run=False), + } + + logs = tuple( + module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + + def test_collect_configuration_run_summary_logs_extract_with_repository_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise( ValueError