From 63198088c4ab972ca5cd50dc45cafdd23f6e0966 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 9 Jan 2024 13:47:20 -0800 Subject: [PATCH] Store included configuration files within each backup archive in support of the "config bootstrap" action (#736). --- NEWS | 2 + borgmatic/actions/config/bootstrap.py | 16 +-- borgmatic/actions/create.py | 6 +- borgmatic/borg/create.py | 12 +- borgmatic/commands/borgmatic.py | 43 +++--- borgmatic/config/generate.py | 3 +- borgmatic/config/load.py | 64 +++++---- borgmatic/config/validate.py | 10 +- docs/how-to/extract-a-backup.md | 5 + tests/integration/config/test_load.py | 83 ++++++++--- tests/integration/config/test_validate.py | 21 ++- tests/unit/actions/test_create.py | 15 +- tests/unit/borg/test_create.py | 159 ++++++++++++++-------- tests/unit/commands/test_borgmatic.py | 136 ++++++++++++------ tests/unit/config/test_load.py | 30 ++-- 15 files changed, 401 insertions(+), 204 deletions(-) diff --git a/NEWS b/NEWS index a7944855..d97daa27 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.8.7.dev0 + * #736: Store included configuration files within each backup archive in support of the "config + bootstrap" action. Previously, only top-level configuration files were stored. * #810: SECURITY: Prevent shell injection attacks within the PostgreSQL hook, the MongoDB hook, the SQLite hook, the "borgmatic borg" action, and command hook variable/constant interpolation. diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 415ba139..77a96f37 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -13,14 +13,11 @@ logger = logging.getLogger(__name__) def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): ''' - Given: - The bootstrap arguments, which include the repository and archive name, borgmatic source directory, - destination directory, and whether to strip components. - The global arguments, which include the dry run flag - and the local borg version, - Return: - The config paths from the manifest.json file in the borgmatic source directory after extracting it from the - repository. + Given the bootstrap arguments as an argparse.Namespace (containing the repository and archive + name, borgmatic source directory, destination directory, and whether to strip components), the + global arguments as an argparse.Namespace (containing the dry run flag and the local borg + version), return the config paths from the manifest.json file in the borgmatic source directory + after extracting it from the repository. Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the expected configuration path data. @@ -32,6 +29,7 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json') ) config = {'ssh_command': bootstrap_arguments.ssh_command} + extract_process = borgmatic.borg.extract.extract_archive( global_arguments.dry_run, bootstrap_arguments.repository, @@ -48,8 +46,8 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): global_arguments, extract_to_stdout=True, ) - manifest_json = extract_process.stdout.read() + if not manifest_json: raise ValueError( 'Cannot read configuration paths from archive due to missing bootstrap manifest' diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 2c9bc646..6d15585e 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -47,6 +47,7 @@ def run_create( config_filename, repository, config, + config_paths, hook_context, local_borg_version, create_arguments, @@ -90,7 +91,9 @@ def run_create( ) if config.get('store_config_files', True): create_borgmatic_manifest( - config, global_arguments.used_config_paths, global_arguments.dry_run + config, + config_paths, + global_arguments.dry_run, ) stream_processes = [process for processes in active_dumps.values() for process in processes] @@ -98,6 +101,7 @@ def run_create( global_arguments.dry_run, repository['path'], config, + config_paths, local_borg_version, global_arguments, local_path=local_path, diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index b09eac8f..9d2a64ce 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -323,6 +323,7 @@ def create_archive( dry_run, repository_path, config, + config_paths, local_borg_version, global_arguments, local_path='borg', @@ -334,8 +335,9 @@ def create_archive( stream_processes=None, ): ''' - Given vebosity/dry-run flags, a local or remote repository path, and a configuration dict, - create a Borg archive and return Borg's JSON output (if any). + Given vebosity/dry-run flags, a local or remote repository path, a configuration dict, a + sequence of loaded configuration paths, the local Borg version, and global arguments as an + argparse.Namespace instance, create a Borg archive and return Borg's JSON output (if any). If a sequence of stream processes is given (instances of subprocess.Popen), then execute the create command while also triggering the given processes to produce output. @@ -351,11 +353,7 @@ def create_archive( expand_directories( tuple(config.get('source_directories', ())) + borgmatic_source_directories - + tuple( - global_arguments.used_config_paths - if config.get('store_config_files', True) - else () - ) + + tuple(config_paths if config.get('store_config_files', True) else ()) ) ), additional_directory_devices=map_directories_to_devices( diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 4e7ecde7..52f00da3 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -58,11 +58,11 @@ def get_skip_actions(config, arguments): return skip_actions -def run_configuration(config_filename, config, arguments): +def run_configuration(config_filename, config, config_paths, arguments): ''' - Given a config filename, the corresponding parsed config dict, and command-line arguments as a - dict from subparser name to a namespace of parsed arguments, execute the defined create, prune, - compact, check, and/or other actions. + Given a config filename, the corresponding parsed config dict, a sequence of loaded + configuration paths, and command-line arguments as a dict from subparser name to a namespace of + parsed arguments, execute the defined create, prune, compact, check, and/or other actions. Yield a combination of: @@ -144,6 +144,7 @@ def run_configuration(config_filename, config, arguments): arguments=arguments, config_filename=config_filename, config=config, + config_paths=config_paths, local_path=local_path, remote_path=remote_path, local_borg_version=local_borg_version, @@ -264,6 +265,7 @@ def run_actions( arguments, config_filename, config, + config_paths, local_path, remote_path, local_borg_version, @@ -271,9 +273,9 @@ def run_actions( ): ''' Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration - filename, several different configuration dicts, local and remote paths to Borg, a local Borg - version string, and a repository name, run all actions from the command-line arguments on the - given repository. + filename, a configuration dict, a sequence of loaded configuration paths, local and remote paths + to Borg, a local Borg version string, and a repository name, run all actions from the + command-line arguments on the given repository. Yield JSON output strings from executing any actions that produce JSON. @@ -328,6 +330,7 @@ def run_actions( config_filename, repository, config, + config_paths, hook_context, local_borg_version, action_arguments, @@ -502,13 +505,15 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): ''' Given a sequence of configuration filenames, load and validate each configuration file. Return the results as a tuple of: dict of configuration filename to corresponding parsed configuration, - and sequence of logging.LogRecord instances containing any parse errors. + a sequence of paths for all loaded configuration files (including includes), and a sequence of + logging.LogRecord instances containing any parse errors. Log records are returned here instead of being logged directly because logging isn't yet initialized at this point! ''' # Dict mapping from config filename to corresponding parsed config dict. configs = collections.OrderedDict() + config_paths = set() logs = [] # Parse and load each configuration file. @@ -525,9 +530,10 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): ] ) try: - configs[config_filename], parse_logs = validate.parse_configuration( + configs[config_filename], paths, parse_logs = validate.parse_configuration( config_filename, validate.schema_filename(), overrides, resolve_env ) + config_paths.update(paths) logs.extend(parse_logs) except PermissionError: logs.extend( @@ -557,7 +563,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): ] ) - return (configs, logs) + return (configs, sorted(config_paths), logs) def log_record(suppress_log=False, **kwargs): @@ -724,12 +730,12 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par return -def collect_configuration_run_summary_logs(configs, arguments): +def collect_configuration_run_summary_logs(configs, config_paths, arguments): ''' - Given a dict of configuration filename to corresponding parsed configuration and parsed - command-line arguments as a dict from subparser name to a parsed namespace of arguments, run - each configuration file and yield a series of logging.LogRecord instances containing summary - information about each run. + Given a dict of configuration filename to corresponding parsed configuration, a sequence of + loaded configuration paths, and parsed command-line arguments as a dict from subparser name to a + parsed namespace of arguments, run each configuration file and yield a series of + logging.LogRecord instances containing summary information about each run. As a side effect of running through these configuration files, output their JSON results, if any, to stdout. @@ -774,7 +780,7 @@ def collect_configuration_run_summary_logs(configs, arguments): # Execute the actions corresponding to each configuration file. json_results = [] for config_filename, config in configs.items(): - results = list(run_configuration(config_filename, config, arguments)) + results = list(run_configuration(config_filename, config, config_paths, arguments)) error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord)) if error_logs: @@ -855,8 +861,7 @@ def main(extra_summary_logs=[]): # pragma: no cover sys.exit(0) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) - global_arguments.used_config_paths = list(config_filenames) - configs, parse_logs = load_configurations( + configs, config_paths, parse_logs = load_configurations( config_filenames, global_arguments.overrides, global_arguments.resolve_env ) configuration_parse_errors = ( @@ -893,7 +898,7 @@ def main(extra_summary_logs=[]): # pragma: no cover configs, arguments, configuration_parse_errors ) ) - or list(collect_configuration_run_summary_logs(configs, arguments)) + or list(collect_configuration_run_summary_logs(configs, config_paths, arguments)) ) ) summary_logs_max_level = max(log.levelno for log in summary_logs) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 2977dbe0..ad5eaea3 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -225,8 +225,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi favoring values from the source when there are collisions. The purpose of this is to upgrade configuration files from old versions of borgmatic by adding - new - configuration keys and comments. + new configuration keys and comments. ''' if not source_config: return destination_config diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index e34da8ae..978fc8e4 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -9,18 +9,18 @@ import ruamel.yaml logger = logging.getLogger(__name__) -def probe_and_include_file(filename, include_directories): +def probe_and_include_file(filename, include_directories, config_paths): ''' - Given a filename to include and a list of include directories to search for matching files, - probe for the file, load it, and return the loaded configuration as a data structure of nested - dicts, lists, etc. + Given a filename to include, a list of include directories to search for matching files, and a + set of configuration paths, probe for the file, load it, and return the loaded configuration as + a data structure of nested dicts, lists, etc. Add the filename to the given configuration paths. Raise FileNotFoundError if the included file was not found. ''' expanded_filename = os.path.expanduser(filename) if os.path.isabs(expanded_filename): - return load_configuration(expanded_filename) + return load_configuration(expanded_filename, config_paths) candidate_filenames = { os.path.join(directory, expanded_filename) for directory in include_directories @@ -28,32 +28,33 @@ def probe_and_include_file(filename, include_directories): for candidate_filename in candidate_filenames: if os.path.exists(candidate_filename): - return load_configuration(candidate_filename) + return load_configuration(candidate_filename, config_paths) raise FileNotFoundError( f'Could not find include {filename} at {" or ".join(candidate_filenames)}' ) -def include_configuration(loader, filename_node, include_directory): +def include_configuration(loader, filename_node, include_directory, config_paths): ''' Given a ruamel.yaml.loader.Loader, a ruamel.yaml.nodes.ScalarNode containing the included - filename (or a list containing multiple such filenames), and an include directory path to search - for matching files, load the given YAML filenames (ignoring the given loader so we can use our - own) and return their contents as data structure of nested dicts, lists, etc. If the given + filename (or a list containing multiple such filenames), an include directory path to search for + matching files, and a set of configuration paths, load the given YAML filenames (ignoring the + given loader so we can use our own) and return their contents as data structure of nested dicts, + lists, etc. Add the names of included files to the given configuration paths. If the given filename node's value is a scalar string, then the return value will be a single value. But if the given node value is a list, then the return value will be a list of values, one per loaded configuration file. - If a filename is relative, probe for it within 1. the current working directory and 2. the given - include directory. + If a filename is relative, probe for it within: 1. the current working directory and 2. the + given include directory. Raise FileNotFoundError if an included file was not found. ''' include_directories = [os.getcwd(), os.path.abspath(include_directory)] if isinstance(filename_node.value, str): - return probe_and_include_file(filename_node.value, include_directories) + return probe_and_include_file(filename_node.value, include_directories, config_paths) if ( isinstance(filename_node.value, list) @@ -63,7 +64,7 @@ def include_configuration(loader, filename_node, include_directory): # Reversing the values ensures the correct ordering if these includes are subsequently # merged together. return [ - probe_and_include_file(node.value, include_directories) + probe_and_include_file(node.value, include_directories, config_paths) for node in reversed(filename_node.value) ] @@ -109,11 +110,17 @@ class Include_constructor(ruamel.yaml.SafeConstructor): separate YAML configuration files. Example syntax: `option: !include common.yaml` ''' - def __init__(self, preserve_quotes=None, loader=None, include_directory=None): + def __init__( + self, preserve_quotes=None, loader=None, include_directory=None, config_paths=None + ): super(Include_constructor, self).__init__(preserve_quotes, loader) self.add_constructor( '!include', - functools.partial(include_configuration, include_directory=include_directory), + functools.partial( + include_configuration, + include_directory=include_directory, + config_paths=config_paths, + ), ) # These are catch-all error handlers for tags that don't get applied and removed by @@ -155,26 +162,33 @@ class Include_constructor(ruamel.yaml.SafeConstructor): node.value = deep_merge_nodes(node.value) -def load_configuration(filename): +def load_configuration(filename, config_paths=None): ''' Load the given configuration file and return its contents as a data structure of nested dicts - and lists. + and lists. Add the filename to the given configuration paths set, and also add any included + configuration filenames. Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError if there are too many recursive includes. ''' + if config_paths is None: + config_paths = set() - # Use an embedded derived class for the include constructor so as to capture the filename - # value. (functools.partial doesn't work for this use case because yaml.Constructor has to be - # an actual class.) - class Include_constructor_with_include_directory(Include_constructor): + # Use an embedded derived class for the include constructor so as to capture the include + # directory and configuration paths values. (functools.partial doesn't work for this use case + # because yaml.Constructor has to be an actual class.) + class Include_constructor_with_extras(Include_constructor): def __init__(self, preserve_quotes=None, loader=None): - super(Include_constructor_with_include_directory, self).__init__( - preserve_quotes, loader, include_directory=os.path.dirname(filename) + super(Include_constructor_with_extras, self).__init__( + preserve_quotes, + loader, + include_directory=os.path.dirname(filename), + config_paths=config_paths, ) yaml = ruamel.yaml.YAML(typ='safe') - yaml.Constructor = Include_constructor_with_include_directory + yaml.Constructor = Include_constructor_with_extras + config_paths.add(filename) with open(filename) as file: return yaml.load(file.read()) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 17f39a92..3ac205e8 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -97,14 +97,16 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv 'checks': ['repository', 'archives'], } - Also return a sequence of logging.LogRecord instances containing any warnings about the - configuration. + Also return a set of loaded configuration paths and a sequence of logging.LogRecord instances + containing any warnings about the configuration. Raise FileNotFoundError if the file does not exist, PermissionError if the user does not have permissions to read the file, or Validation_error if the config does not match the schema. ''' + config_paths = set() + try: - config = load.load_configuration(config_filename) + config = load.load_configuration(config_filename, config_paths) schema = load.load_configuration(schema_filename) except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) @@ -130,7 +132,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv apply_logical_validation(config_filename, config) - return config, logs + return config, config_paths, logs def normalize_repository_path(repository): diff --git a/docs/how-to/extract-a-backup.md b/docs/how-to/extract-a-backup.md index 2517f412..6e0d04b8 100644 --- a/docs/how-to/extract-a-backup.md +++ b/docs/how-to/extract-a-backup.md @@ -191,3 +191,8 @@ for bootstrapping. borgmatic configuration files, for instance if they contain sensitive information you don't want to store even inside your encrypted backups. If you do this though, the `config bootstrap` action will no longer work. + +New in version 1.8.7 Included +configuration files are stored in each backup archive. This means that the +`config bootstrap` action not only extracts the top-level configuration files +but also the includes they depend upon. diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index 73dde4dc..e13c69c8 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -12,7 +12,10 @@ def test_load_configuration_parses_contents(): config_file = io.StringIO('key: value') config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) - assert module.load_configuration('config.yaml') == {'key': 'value'} + config_paths = {'other.yaml'} + + assert module.load_configuration('config.yaml', config_paths) == {'key': 'value'} + assert config_paths == {'config.yaml', 'other.yaml'} def test_load_configuration_with_only_integer_value_does_not_raise(): @@ -20,7 +23,10 @@ def test_load_configuration_with_only_integer_value_does_not_raise(): config_file = io.StringIO('33') config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) - assert module.load_configuration('config.yaml') == 33 + config_paths = {'other.yaml'} + + assert module.load_configuration('config.yaml', config_paths) == 33 + assert config_paths == {'config.yaml', 'other.yaml'} def test_load_configuration_inlines_include_relative_to_current_directory(): @@ -34,8 +40,10 @@ def test_load_configuration_inlines_include_relative_to_current_directory(): config_file = io.StringIO('key: !include include.yaml') config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = {'other.yaml'} - assert module.load_configuration('config.yaml') == {'key': 'value'} + assert module.load_configuration('config.yaml', config_paths) == {'key': 'value'} + assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'} def test_load_configuration_inlines_include_relative_to_config_parent_directory(): @@ -56,8 +64,10 @@ def test_load_configuration_inlines_include_relative_to_config_parent_directory( config_file = io.StringIO('key: !include include.yaml') config_file.name = '/etc/config.yaml' builtins.should_receive('open').with_args('/etc/config.yaml').and_return(config_file) + config_paths = {'other.yaml'} - assert module.load_configuration('/etc/config.yaml') == {'key': 'value'} + assert module.load_configuration('/etc/config.yaml', config_paths) == {'key': 'value'} + assert config_paths == {'/etc/config.yaml', '/etc/include.yaml', 'other.yaml'} def test_load_configuration_raises_if_relative_include_does_not_exist(): @@ -70,9 +80,10 @@ def test_load_configuration_raises_if_relative_include_does_not_exist(): config_file = io.StringIO('key: !include include.yaml') config_file.name = '/etc/config.yaml' builtins.should_receive('open').with_args('/etc/config.yaml').and_return(config_file) + config_paths = set() with pytest.raises(FileNotFoundError): - module.load_configuration('/etc/config.yaml') + module.load_configuration('/etc/config.yaml', config_paths) def test_load_configuration_inlines_absolute_include(): @@ -86,8 +97,10 @@ def test_load_configuration_inlines_absolute_include(): config_file = io.StringIO('key: !include /root/include.yaml') config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = {'other.yaml'} - assert module.load_configuration('config.yaml') == {'key': 'value'} + assert module.load_configuration('config.yaml', config_paths) == {'key': 'value'} + assert config_paths == {'config.yaml', '/root/include.yaml', 'other.yaml'} def test_load_configuration_raises_if_absolute_include_does_not_exist(): @@ -98,9 +111,10 @@ def test_load_configuration_raises_if_absolute_include_does_not_exist(): config_file = io.StringIO('key: !include /root/include.yaml') config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = set() with pytest.raises(FileNotFoundError): - assert module.load_configuration('config.yaml') + assert module.load_configuration('config.yaml', config_paths) def test_load_configuration_inlines_multiple_file_include_as_list(): @@ -117,8 +131,15 @@ def test_load_configuration_inlines_multiple_file_include_as_list(): config_file = io.StringIO('key: !include [/root/include1.yaml, /root/include2.yaml]') config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = {'other.yaml'} - assert module.load_configuration('config.yaml') == {'key': ['value2', 'value1']} + assert module.load_configuration('config.yaml', config_paths) == {'key': ['value2', 'value1']} + assert config_paths == { + 'config.yaml', + '/root/include1.yaml', + '/root/include2.yaml', + 'other.yaml', + } def test_load_configuration_include_with_unsupported_filename_type_raises(): @@ -129,9 +150,10 @@ def test_load_configuration_include_with_unsupported_filename_type_raises(): config_file = io.StringIO('key: !include {path: /root/include.yaml}') config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = set() with pytest.raises(ValueError): - module.load_configuration('config.yaml') + module.load_configuration('config.yaml', config_paths) def test_load_configuration_merges_include(): @@ -155,8 +177,13 @@ def test_load_configuration_merges_include(): ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = {'other.yaml'} - assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'} + assert module.load_configuration('config.yaml', config_paths) == { + 'foo': 'override', + 'baz': 'quux', + } + assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'} def test_load_configuration_merges_multiple_file_include(): @@ -188,12 +215,14 @@ def test_load_configuration_merges_multiple_file_include(): ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = {'other.yaml'} - assert module.load_configuration('config.yaml') == { + assert module.load_configuration('config.yaml', config_paths) == { 'foo': 'override', 'baz': 'second', 'original': 'yes', } + assert config_paths == {'config.yaml', '/tmp/include1.yaml', '/tmp/include2.yaml', 'other.yaml'} def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values(): @@ -226,11 +255,13 @@ def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_value ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = {'other.yaml'} - assert module.load_configuration('config.yaml') == { + assert module.load_configuration('config.yaml', config_paths) == { 'stuff': {'foo': 'override'}, 'other': {'a': 'override', 'c': 'd'}, } + assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'} def test_load_configuration_with_retain_tag_but_without_merge_include_raises(): @@ -256,9 +287,10 @@ def test_load_configuration_with_retain_tag_but_without_merge_include_raises(): ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = set() with pytest.raises(ValueError): - module.load_configuration('config.yaml') + module.load_configuration('config.yaml', config_paths) def test_load_configuration_with_retain_tag_on_scalar_raises(): @@ -284,9 +316,10 @@ def test_load_configuration_with_retain_tag_on_scalar_raises(): ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = set() with pytest.raises(ValueError): - module.load_configuration('config.yaml') + module.load_configuration('config.yaml', config_paths) def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values(): @@ -315,8 +348,10 @@ def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_val ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = {'other.yaml'} - assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']} + assert module.load_configuration('config.yaml', config_paths) == {'stuff': ['a', 'c', 'x', 'y']} + assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'} def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise(): @@ -345,8 +380,12 @@ def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_do ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = {'other.yaml'} - assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']} + assert module.load_configuration('config.yaml', config_paths) == { + 'stuff': ['a', 'b', 'c', 'x', 'y'] + } + assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'} def test_load_configuration_with_omit_tag_on_non_list_item_raises(): @@ -374,9 +413,10 @@ def test_load_configuration_with_omit_tag_on_non_list_item_raises(): ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = set() with pytest.raises(ValueError): - module.load_configuration('config.yaml') + module.load_configuration('config.yaml', config_paths) def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises(): @@ -403,9 +443,10 @@ def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises(): ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = set() with pytest.raises(ValueError): - module.load_configuration('config.yaml') + module.load_configuration('config.yaml', config_paths) def test_load_configuration_with_omit_tag_but_without_merge_raises(): @@ -433,9 +474,10 @@ def test_load_configuration_with_omit_tag_but_without_merge_raises(): ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = set() with pytest.raises(ValueError): - module.load_configuration('config.yaml') + module.load_configuration('config.yaml', config_paths) def test_load_configuration_does_not_merge_include_list(): @@ -460,9 +502,10 @@ def test_load_configuration_does_not_merge_include_list(): ) config_file.name = 'config.yaml' builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = set() with pytest.raises(module.ruamel.yaml.error.YAMLError): - assert module.load_configuration('config.yaml') + assert module.load_configuration('config.yaml', config_paths) @pytest.mark.parametrize( diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 92be96da..e438b480 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -58,7 +58,7 @@ def test_parse_configuration_transforms_file_into_mapping(): ''' ) - config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { 'source_directories': ['/home', '/etc'], @@ -68,6 +68,7 @@ def test_parse_configuration_transforms_file_into_mapping(): 'keep_minutely': 60, 'checks': [{'name': 'repository'}, {'name': 'archives'}], } + assert config_paths == {'/tmp/config.yaml'} assert logs == [] @@ -84,12 +85,13 @@ def test_parse_configuration_passes_through_quoted_punctuation(): ''' ) - config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { 'source_directories': [f'/home/{string.punctuation}'], 'repositories': [{'path': 'test.borg'}], } + assert config_paths == {'/tmp/config.yaml'} assert logs == [] @@ -141,7 +143,7 @@ def test_parse_configuration_inlines_include_inside_deprecated_section(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { 'source_directories': ['/home'], @@ -149,6 +151,7 @@ def test_parse_configuration_inlines_include_inside_deprecated_section(): 'keep_daily': 7, 'keep_hourly': 24, } + assert config_paths == {'/tmp/include.yaml', '/tmp/config.yaml'} assert len(logs) == 1 @@ -175,7 +178,7 @@ def test_parse_configuration_merges_include(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { 'source_directories': ['/home'], @@ -183,6 +186,7 @@ def test_parse_configuration_merges_include(): 'keep_daily': 1, 'keep_hourly': 24, } + assert config_paths == {'/tmp/include.yaml', '/tmp/config.yaml'} assert logs == [] @@ -194,6 +198,9 @@ def test_parse_configuration_raises_for_missing_config_file(): def test_parse_configuration_raises_for_missing_schema_file(): mock_config_and_schema('') builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/tmp/config.yaml').and_return( + io.StringIO('foo: bar') + ) builtins.should_receive('open').with_args('/tmp/schema.yaml').and_raise(FileNotFoundError) with pytest.raises(FileNotFoundError): @@ -233,7 +240,7 @@ def test_parse_configuration_applies_overrides(): ''' ) - config, logs = module.parse_configuration( + config, config_paths, logs = module.parse_configuration( '/tmp/config.yaml', '/tmp/schema.yaml', overrides=['location.local_path=borg2'] ) @@ -242,6 +249,7 @@ def test_parse_configuration_applies_overrides(): 'repositories': [{'path': 'hostname.borg'}], 'local_path': 'borg2', } + assert config_paths == {'/tmp/config.yaml'} assert logs == [] @@ -260,11 +268,12 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in ) flexmock(os).should_receive('getenv').replace_with(lambda variable_name, default: default) - config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') assert config == { 'source_directories': ['/home'], 'repositories': [{'path': 'ssh://user@hostname/./repo'}], 'exclude_if_present': ['.nobackup'], } + assert config_paths == {'/tmp/config.yaml'} assert logs diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 948c5190..824bd9ab 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -22,13 +22,14 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): json=False, list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) list( module.run_create( config_filename='test.yaml', repository={'path': 'repo'}, config={}, + config_paths=['/tmp/test.yaml'], hook_context={}, local_borg_version=None, create_arguments=create_arguments, @@ -57,13 +58,14 @@ def test_run_create_with_store_config_files_false_does_not_create_borgmatic_mani json=False, list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) list( module.run_create( config_filename='test.yaml', repository={'path': 'repo'}, config={'store_config_files': False}, + config_paths=['/tmp/test.yaml'], hook_context={}, local_borg_version=None, create_arguments=create_arguments, @@ -94,13 +96,14 @@ def test_run_create_runs_with_selected_repository(): json=False, list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) list( module.run_create( config_filename='test.yaml', repository={'path': 'repo'}, config={}, + config_paths=['/tmp/test.yaml'], hook_context={}, local_borg_version=None, create_arguments=create_arguments, @@ -126,13 +129,14 @@ def test_run_create_bails_if_repository_does_not_match(): json=False, list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) list( module.run_create( config_filename='test.yaml', repository='repo', config={}, + config_paths=['/tmp/test.yaml'], hook_context={}, local_borg_version=None, create_arguments=create_arguments, @@ -167,13 +171,14 @@ def test_run_create_produces_json(): json=True, list_files=flexmock(), ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) assert list( module.run_create( config_filename='test.yaml', repository={'path': 'repo'}, config={}, + config_paths=['/tmp/test.yaml'], hook_context={}, local_borg_version=None, create_arguments=create_arguments, diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index e4f12842..32ef90d3 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -506,8 +506,9 @@ def test_create_archive_calls_borg_with_parameters(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -549,8 +550,9 @@ def test_create_archive_calls_borg_with_environment(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -594,23 +596,24 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert 'repositories': ['repo'], 'patterns': ['pattern'], }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) -def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sources_and_config_paths(): +def test_create_archive_with_sources_and_config_paths_calls_borg_with_sources_and_config_paths(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return( - ('foo', 'bar', '/etc/borgmatic/config.yaml') + ('foo', 'bar', '/tmp/test.yaml') ) flexmock(module).should_receive('map_directories_to_devices').and_return({}) flexmock(module).should_receive('expand_directories').with_args([]).and_return(()) flexmock(module).should_receive('expand_directories').with_args( - ('foo', 'bar', '/etc/borgmatic/config.yaml') - ).and_return(('foo', 'bar', '/etc/borgmatic/config.yaml')) + ('foo', 'bar', '/tmp/test.yaml') + ).and_return(('foo', 'bar', '/tmp/test.yaml')) flexmock(module).should_receive('expand_directories').with_args([]).and_return(()) flexmock(module).should_receive('pattern_root_directories').and_return([]) flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) @@ -627,7 +630,7 @@ def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sourc environment = {'BORG_THINGY': 'YUP'} flexmock(module.environment).should_receive('make_environment').and_return(environment) flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('/etc/borgmatic/config.yaml',), + ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('/tmp/test.yaml',), output_log_level=logging.INFO, output_file=None, borg_local_path='borg', @@ -642,12 +645,13 @@ def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sourc 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']), + global_arguments=flexmock(log_json=False), ) -def test_create_archive_with_sources_and_used_config_paths_with_store_config_files_false_calls_borg_with_sources_and_no_config_paths(): +def test_create_archive_with_sources_and_config_paths_with_store_config_files_false_calls_borg_with_sources_and_no_config_paths(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) @@ -689,8 +693,9 @@ def test_create_archive_with_sources_and_used_config_paths_with_store_config_fil 'repositories': ['repo'], 'store_config_files': False, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']), + global_arguments=flexmock(log_json=False), ) @@ -734,8 +739,9 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): 'repositories': ['repo'], 'exclude_patterns': ['exclude'], }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -777,8 +783,9 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -818,8 +825,9 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), json=True, ) @@ -862,8 +870,9 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -903,8 +912,9 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), json=True, ) @@ -946,8 +956,9 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -991,8 +1002,9 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), stats=True, ) @@ -1035,8 +1047,9 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte 'exclude_patterns': None, 'checkpoint_interval': 600, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1078,8 +1091,9 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume 'exclude_patterns': None, 'checkpoint_volume': 1024, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1121,8 +1135,9 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param 'exclude_patterns': None, 'chunker_params': '1,2,3,4', }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1164,8 +1179,9 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( 'exclude_patterns': None, 'compression': 'rle', }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1213,8 +1229,9 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ 'exclude_patterns': None, 'upload_rate_limit': 100, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1258,8 +1275,9 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory 'working_directory': '/working/dir', 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1301,8 +1319,9 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par 'one_file_system': True, 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1350,8 +1369,9 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( 'numeric_ids': True, 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1403,8 +1423,9 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter 'read_special': True, 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1458,8 +1479,9 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete option_name: option_value, 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1512,8 +1534,9 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete 'atime': option_value, 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1566,8 +1589,9 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete 'flags': option_value, 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1609,8 +1633,9 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( 'files_cache': 'ctime,size', 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1651,8 +1676,9 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -1694,8 +1720,9 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -1738,8 +1765,9 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): 'exclude_patterns': None, 'umask': 740, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1780,8 +1808,9 @@ def test_create_archive_with_log_json_calls_borg_with_log_json_parameters(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=True, used_config_paths=[]), + global_arguments=flexmock(log_json=True), ) @@ -1823,8 +1852,9 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): 'exclude_patterns': None, 'lock_wait': 5, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -1865,8 +1895,9 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), stats=True, ) @@ -1908,8 +1939,9 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), list_files=True, ) @@ -1952,8 +1984,9 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), progress=True, ) @@ -1995,8 +2028,9 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), progress=True, ) @@ -2057,8 +2091,9 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), progress=True, stream_processes=processes, ) @@ -2121,8 +2156,9 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log 'exclude_patterns': None, 'read_special': False, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2188,8 +2224,9 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2252,8 +2289,9 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec 'exclude_patterns': None, 'read_special': True, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2293,8 +2331,9 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), json=True, ) @@ -2336,8 +2375,9 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), json=True, stats=True, ) @@ -2383,8 +2423,9 @@ def test_create_archive_with_source_directories_glob_expands(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -2426,8 +2467,9 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -2468,8 +2510,9 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -2511,8 +2554,9 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): 'exclude_patterns': None, 'archive_name_format': 'ARCHIVE_NAME', }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -2555,8 +2599,9 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): 'exclude_patterns': None, 'archive_name_format': 'Documents_{hostname}-{now}', # noqa: FS003 }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -2599,8 +2644,9 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): 'exclude_patterns': None, 'archive_name_format': 'Documents_{hostname}-{now}', # noqa: FS003 }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -2642,8 +2688,9 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): 'exclude_patterns': None, 'extra_borg_options': {'create': '--extra --options'}, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) @@ -2702,8 +2749,9 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read 'repositories': ['repo'], 'exclude_patterns': None, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2727,8 +2775,9 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ 'exclude_patterns': None, 'source_directories_must_exist': True, }, + config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False, used_config_paths=[]), + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 85b25d7d..eae4d110 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -38,7 +38,7 @@ def test_run_configuration_runs_actions_for_each_repository(): config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} arguments = {'global': flexmock(monitoring_verbosity=1)} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == expected_results @@ -51,7 +51,7 @@ def test_run_configuration_with_skip_actions_does_not_raise(): config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'skip_actions': ['compact']} arguments = {'global': flexmock(monitoring_verbosity=1)} - list(module.run_configuration('test.yaml', config, arguments)) + list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) def test_run_configuration_with_invalid_borg_version_errors(): @@ -64,7 +64,7 @@ def test_run_configuration_with_invalid_borg_version_errors(): config = {'repositories': ['foo']} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock()} - list(module.run_configuration('test.yaml', config, arguments)) + list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) def test_run_configuration_logs_monitor_start_error(): @@ -80,7 +80,7 @@ def test_run_configuration_logs_monitor_start_error(): config = {'repositories': ['foo']} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == expected_results @@ -96,7 +96,7 @@ def test_run_configuration_bails_for_monitor_start_soft_failure(): config = {'repositories': ['foo']} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == [] @@ -113,7 +113,7 @@ def test_run_configuration_logs_actions_error(): config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == expected_results @@ -130,7 +130,7 @@ def test_run_configuration_bails_for_actions_soft_failure(): config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == [] @@ -148,7 +148,7 @@ def test_run_configuration_logs_monitor_log_error(): config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == expected_results @@ -167,7 +167,7 @@ def test_run_configuration_bails_for_monitor_log_soft_failure(): config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == [] @@ -185,7 +185,7 @@ def test_run_configuration_logs_monitor_finish_error(): config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == expected_results @@ -204,7 +204,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == [] @@ -219,7 +219,7 @@ def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_ar config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=-2, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == [] @@ -236,7 +236,7 @@ def test_run_configuration_logs_on_error_hook_error(): config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == expected_results @@ -253,7 +253,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == expected_results @@ -268,7 +268,7 @@ def test_run_configuration_retries_soft_error(): flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once() config = {'repositories': [{'path': 'foo'}], 'retries': 1} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == [] @@ -292,7 +292,7 @@ def test_run_configuration_retries_hard_error(): ).and_return(error_logs) config = {'repositories': [{'path': 'foo'}], 'retries': 1} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == error_logs @@ -311,7 +311,7 @@ def test_run_configuration_repos_ordered(): ).and_return(expected_results[1:]).ordered() config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == expected_results @@ -346,7 +346,7 @@ def test_run_configuration_retries_round_robin(): 'retries': 1, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == foo_error_logs + bar_error_logs @@ -379,7 +379,7 @@ def test_run_configuration_retries_one_passes(): 'retries': 1, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == error_logs @@ -423,7 +423,7 @@ def test_run_configuration_retry_wait(): 'retry_wait': 10, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == error_logs @@ -463,7 +463,7 @@ def test_run_configuration_retries_timeout_multiple_repos(): 'retry_wait': 10, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - results = list(module.run_configuration('test.yaml', config, arguments)) + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == error_logs @@ -478,6 +478,7 @@ def test_run_actions_runs_rcreate(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -495,6 +496,7 @@ def test_run_actions_adds_log_file_to_hook_context(): config_filename=object, repository={'path': 'repo'}, config={'repositories': []}, + config_paths=[], hook_context={'repository': 'repo', 'repositories': '', 'log_file': 'foo'}, local_borg_version=object, create_arguments=object, @@ -509,6 +511,7 @@ def test_run_actions_adds_log_file_to_hook_context(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -529,6 +532,7 @@ def test_run_actions_runs_transfer(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'transfer': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -549,6 +553,7 @@ def test_run_actions_runs_create(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -569,6 +574,7 @@ def test_run_actions_with_skip_actions_skips_create(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, config_filename=flexmock(), config={'repositories': [], 'skip_actions': ['create']}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -588,6 +594,7 @@ def test_run_actions_runs_prune(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'prune': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -607,6 +614,7 @@ def test_run_actions_with_skip_actions_skips_prune(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'prune': flexmock()}, config_filename=flexmock(), config={'repositories': [], 'skip_actions': ['prune']}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -626,6 +634,7 @@ def test_run_actions_runs_compact(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'compact': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -645,6 +654,7 @@ def test_run_actions_with_skip_actions_skips_compact(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'compact': flexmock()}, config_filename=flexmock(), config={'repositories': [], 'skip_actions': ['compact']}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -665,6 +675,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -685,6 +696,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -705,6 +717,7 @@ def test_run_actions_with_skip_actions_skips_check(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, config_filename=flexmock(), config={'repositories': [], 'skip_actions': ['check']}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -724,6 +737,7 @@ def test_run_actions_runs_extract(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'extract': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -743,6 +757,7 @@ def test_run_actions_runs_export_tar(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'export-tar': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -762,6 +777,7 @@ def test_run_actions_runs_mount(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'mount': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -781,6 +797,7 @@ def test_run_actions_runs_restore(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'restore': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -801,6 +818,7 @@ def test_run_actions_runs_rlist(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rlist': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -822,6 +840,7 @@ def test_run_actions_runs_list(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'list': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -843,6 +862,7 @@ def test_run_actions_runs_rinfo(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rinfo': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -864,6 +884,7 @@ def test_run_actions_runs_info(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'info': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -884,6 +905,7 @@ def test_run_actions_runs_break_lock(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'break-lock': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -903,6 +925,7 @@ def test_run_actions_runs_export_key(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'export': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -922,6 +945,7 @@ def test_run_actions_runs_borg(): arguments={'global': flexmock(dry_run=False, log_file='foo'), 'borg': flexmock()}, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -946,6 +970,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): }, config_filename=flexmock(), config={'repositories': []}, + config_paths=[], local_path=flexmock(), remote_path=flexmock(), local_borg_version=flexmock(), @@ -960,30 +985,33 @@ def test_load_configurations_collects_parsed_configurations_and_logs(): test_expected_logs = [flexmock(), flexmock()] other_expected_logs = [flexmock(), flexmock()] flexmock(module.validate).should_receive('parse_configuration').and_return( - configuration, test_expected_logs - ).and_return(other_configuration, other_expected_logs) + configuration, ['/tmp/test.yaml'], test_expected_logs + ).and_return(other_configuration, ['/tmp/other.yaml'], other_expected_logs) - configs, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml'))) + configs, config_paths, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml'))) assert configs == {'test.yaml': configuration, 'other.yaml': other_configuration} + assert config_paths == ['/tmp/other.yaml', '/tmp/test.yaml'] assert set(logs) >= set(test_expected_logs + other_expected_logs) def test_load_configurations_logs_warning_for_permission_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(PermissionError) - configs, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) assert configs == {} + assert config_paths == [] assert max(log.levelno for log in logs) == logging.WARNING def test_load_configurations_logs_critical_for_parse_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError) - configs, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) assert configs == {} + assert config_paths == [] assert max(log.levelno for log in logs) == logging.CRITICAL @@ -1214,7 +1242,9 @@ def test_collect_configuration_run_summary_logs_info_for_success(): arguments = {} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert {log.levelno for log in logs} == {logging.INFO} @@ -1226,7 +1256,9 @@ def test_collect_configuration_run_summary_executes_hooks_for_create(): arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert {log.levelno for log in logs} == {logging.INFO} @@ -1239,7 +1271,9 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): arguments = {'extract': flexmock(repository='repo')} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert {log.levelno for log in logs} == {logging.INFO} @@ -1254,7 +1288,9 @@ def test_collect_configuration_run_summary_logs_extract_with_repository_error(): arguments = {'extract': flexmock(repository='repo')} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert logs == expected_logs @@ -1267,7 +1303,9 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_mount(): arguments = {'mount': flexmock(repository='repo')} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert {log.levelno for log in logs} == {logging.INFO} @@ -1282,7 +1320,9 @@ def test_collect_configuration_run_summary_logs_mount_with_repository_error(): arguments = {'mount': flexmock(repository='repo')} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert logs == expected_logs @@ -1293,7 +1333,9 @@ def test_collect_configuration_run_summary_logs_missing_configs_error(): expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - logs = tuple(module.collect_configuration_run_summary_logs({}, arguments=arguments)) + logs = tuple( + module.collect_configuration_run_summary_logs({}, config_paths=[], arguments=arguments) + ) assert logs == expected_logs @@ -1305,7 +1347,9 @@ def test_collect_configuration_run_summary_logs_pre_hook_error(): arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert logs == expected_logs @@ -1320,7 +1364,9 @@ def test_collect_configuration_run_summary_logs_post_hook_error(): arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert expected_logs[0] in logs @@ -1335,7 +1381,9 @@ def test_collect_configuration_run_summary_logs_for_list_with_archive_and_reposi arguments = {'list': flexmock(repository='repo', archive='test')} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert logs == expected_logs @@ -1347,7 +1395,9 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list(): arguments = {'list': flexmock(repository='repo', archive=None)} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert {log.levelno for log in logs} == {logging.INFO} @@ -1362,7 +1412,9 @@ def test_collect_configuration_run_summary_logs_run_configuration_error(): arguments = {} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert {log.levelno for log in logs} == {logging.CRITICAL} @@ -1378,7 +1430,9 @@ def test_collect_configuration_run_summary_logs_run_umount_error(): arguments = {'umount': flexmock(mount_point='/mnt')} logs = tuple( - module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) ) assert {log.levelno for log in logs} == {logging.INFO, logging.CRITICAL} @@ -1396,6 +1450,8 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): tuple( module.collect_configuration_run_summary_logs( - {'test.yaml': {}, 'test2.yaml': {}}, arguments=arguments + {'test.yaml': {}, 'test2.yaml': {}}, + config_paths=['/tmp/test.yaml', '/tmp/test2.yaml'], + arguments=arguments, ) ) diff --git a/tests/unit/config/test_load.py b/tests/unit/config/test_load.py index d521146a..4c1d221c 100644 --- a/tests/unit/config/test_load.py +++ b/tests/unit/config/test_load.py @@ -6,27 +6,35 @@ from borgmatic.config import load as module def test_probe_and_include_file_with_absolute_path_skips_probing(): config = flexmock() - flexmock(module).should_receive('load_configuration').with_args('/etc/include.yaml').and_return( - config - ).once() + config_paths = set() + flexmock(module).should_receive('load_configuration').with_args( + '/etc/include.yaml', config_paths + ).and_return(config).once() - assert module.probe_and_include_file('/etc/include.yaml', ['/etc', '/var']) == config + assert ( + module.probe_and_include_file('/etc/include.yaml', ['/etc', '/var'], config_paths) == config + ) def test_probe_and_include_file_with_relative_path_probes_include_directories(): - config = flexmock() + config = {'foo': 'bar'} + config_paths = set() flexmock(module.os.path).should_receive('exists').with_args('/etc/include.yaml').and_return( False ) flexmock(module.os.path).should_receive('exists').with_args('/var/include.yaml').and_return( True ) - flexmock(module).should_receive('load_configuration').with_args('/etc/include.yaml').never() - flexmock(module).should_receive('load_configuration').with_args('/var/include.yaml').and_return( - config - ).once() + flexmock(module).should_receive('load_configuration').with_args( + '/etc/include.yaml', config_paths + ).never() + flexmock(module).should_receive('load_configuration').with_args( + '/var/include.yaml', config_paths + ).and_return(config).once() - assert module.probe_and_include_file('include.yaml', ['/etc', '/var']) == config + assert module.probe_and_include_file('include.yaml', ['/etc', '/var'], config_paths) == { + 'foo': 'bar', + } def test_probe_and_include_file_with_relative_path_and_missing_files_raises(): @@ -34,4 +42,4 @@ def test_probe_and_include_file_with_relative_path_and_missing_files_raises(): flexmock(module).should_receive('load_configuration').never() with pytest.raises(FileNotFoundError): - module.probe_and_include_file('include.yaml', ['/etc', '/var']) + module.probe_and_include_file('include.yaml', ['/etc', '/var'], config_paths=set())