From 689643e5fa0a72268fb3013173f10e4c091671fc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 24 Nov 2024 16:00:33 -0800 Subject: [PATCH] Move bootstrap manifest file creation into a hook so it can actually clean up after itself. --- NEWS | 5 + borgmatic/actions/check.py | 2 +- borgmatic/actions/create.py | 52 ++------- borgmatic/config/normalize.py | 19 ++++ borgmatic/config/schema.yaml | 21 ++-- borgmatic/hooks/dispatch.py | 2 + borgmatic/hooks/dump.py | 1 + borgmatic/hooks/mariadb.py | 1 + borgmatic/hooks/mongodb.py | 1 + borgmatic/hooks/mysql.py | 1 + borgmatic/hooks/postgresql.py | 1 + borgmatic/hooks/sqlite.py | 1 + borgmatic/hooks/zfs.py | 29 +++-- docs/how-to/extract-a-backup.md | 30 ++++-- tests/integration/config/test_validate.py | 6 ++ tests/unit/actions/test_create.py | 124 +--------------------- tests/unit/config/test_normalize.py | 6 ++ tests/unit/hooks/test_mariadb.py | 6 ++ tests/unit/hooks/test_mongodb.py | 7 ++ tests/unit/hooks/test_mysql.py | 6 ++ tests/unit/hooks/test_postgresql.py | 12 +++ tests/unit/hooks/test_sqlite.py | 6 ++ tests/unit/hooks/test_zfs.py | 3 + 23 files changed, 148 insertions(+), 194 deletions(-) diff --git a/NEWS b/NEWS index b4cb3adb..88c70cdd 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,11 @@ 1.9.3.dev0 * #261 (beta): Add a ZFS hook for snapshotting and backing up ZFS datasets. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/ + * After it's stored in a Borg archive, remove the manifest file created in support of + the "bootstrap" action. + * Deprecate the "store_config_files" option at the global scope and move it under the "bootstrap" + hook. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive * Add a "--deleted" flag to the "repo-list" action for listing deleted archives that haven't yet been compacted (Borg 2 only). diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 98335f1b..19b10d9a 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -374,7 +374,7 @@ def collect_spot_check_source_paths( repository_path=repository['path'], config=config, source_directories=borgmatic.actions.create.process_source_directories( - config, config_paths=() + config, ), local_borg_version=local_borg_version, global_arguments=global_arguments, diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index c6a2c960..11e2efa7 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -1,7 +1,5 @@ import glob -import importlib.metadata import itertools -import json import logging import os import pathlib @@ -17,32 +15,6 @@ import borgmatic.hooks.dump logger = logging.getLogger(__name__) -def create_borgmatic_manifest(config, config_paths, borgmatic_runtime_directory, dry_run): - ''' - Given a configuration dict, a sequence of config file paths, the borgmatic runtime directory, - and whether this is a dry run, create a borgmatic manifest file to store the paths to the - configuration files used to create the archive. - ''' - if dry_run: - return - - borgmatic_manifest_path = os.path.join( - borgmatic_runtime_directory, 'bootstrap', 'manifest.json' - ) - - if not os.path.exists(borgmatic_manifest_path): - os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True) - - with open(borgmatic_manifest_path, 'w') as config_list_file: - json.dump( - { - 'borgmatic_version': importlib.metadata.version('borgmatic'), - 'config_paths': config_paths, - }, - config_list_file, - ) - - def expand_directory(directory, working_directory): ''' Given a directory path, expand any tilde (representing a user's home directory) and any globs @@ -146,18 +118,15 @@ def pattern_root_directories(patterns=None): ] -def process_source_directories(config, config_paths, source_directories=None): +def process_source_directories(config, source_directories=None): ''' Given a sequence of source directories (either in the source_directories argument or, lacking - that, from config) and a sequence of config paths to append, expand and deduplicate the source - directories, returning the result. + that, from config), expand and deduplicate the source directories, returning the result. ''' working_directory = borgmatic.config.paths.get_working_directory(config) if source_directories is None: - source_directories = tuple(config.get('source_directories', ())) + ( - tuple(config_paths) if config.get('store_config_files', True) else () - ) + source_directories = tuple(config.get('source_directories', ())) return deduplicate_directories( map_directories_to_devices( @@ -221,12 +190,13 @@ def run_create( borgmatic_runtime_directory, global_arguments.dry_run, ) - source_directories = process_source_directories(config, config_paths) + source_directories = process_source_directories(config) active_dumps = borgmatic.hooks.dispatch.call_hooks( 'dump_data_sources', config, repository['path'], borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES, + config_paths, borgmatic_runtime_directory, source_directories, global_arguments.dry_run, @@ -235,19 +205,9 @@ def run_create( # Process source directories again in case any data source hooks updated them. Without this # step, we could end up with duplicate paths that cause Borg to hang when it tries to read # from the same named pipe twice. - source_directories = process_source_directories(config, config_paths, source_directories) + source_directories = process_source_directories(config, source_directories) stream_processes = [process for processes in active_dumps.values() for process in processes] - if config.get('store_config_files', True): - create_borgmatic_manifest( - config, - config_paths, - borgmatic_runtime_directory, - global_arguments.dry_run, - ) - if not global_arguments.dry_run: - source_directories.append(os.path.join(borgmatic_runtime_directory, 'bootstrap')) - json_output = borgmatic.borg.create.create_archive( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 6a6d3202..a869b24a 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -93,6 +93,25 @@ def normalize(config_filename, config): ) config['exclude_if_present'] = [exclude_if_present] + # Unconditionally set the bootstrap hook so that it's enabled by default and config files get + # stored in each Borg archive. + config.setdefault('bootstrap', {}) + + # Move store_config_files from the global scope to the bootstrap hook. + store_config_files = config.get('store_config_files') + if store_config_files is not None: + logs.append( + logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: The store_config_files option has moved under the bootstrap hook. Specifying store_config_files at the global scope is deprecated and support will be removed from a future release.', + ) + ) + ) + del config['store_config_files'] + config['bootstrap']['store_config_files'] = store_config_files + # Upgrade various monitoring hooks from a string to a dict. healthchecks = config.get('healthchecks') if isinstance(healthchecks, str): diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index af604b43..ddf9aaeb 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -229,13 +229,6 @@ properties: create the check records again (and therefore re-run checks). Defaults to $XDG_STATE_HOME or ~/.local/state. example: /var/lib/borgmatic - store_config_files: - type: boolean - description: | - Store configuration files used to create a backup in the backup - itself. Defaults to true. Changing this to false prevents "borgmatic - bootstrap" from extracting configuration files from the backup. - example: false source_directories_must_exist: type: boolean description: | @@ -942,6 +935,20 @@ properties: of them (after any action). example: - "echo Completed actions." + bootstrap: + type: object + properties: + store_config_files: + type: boolean + description: | + Store configuration files used to create a backup inside the + backup itself. Defaults to true. Changing this to false + prevents "borgmatic bootstrap" from extracting configuration + files from the backup. + example: false + description: | + Support for the "borgmatic bootstrap" action, used to extract + borgmatic configuration files from a backup archive. postgresql_databases: type: array items: diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index 05528df8..dd2e74cf 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -2,6 +2,7 @@ import logging from borgmatic.hooks import ( apprise, + bootstrap, cronhub, cronitor, healthchecks, @@ -23,6 +24,7 @@ logger = logging.getLogger(__name__) HOOK_NAME_TO_MODULE = { 'apprise': apprise, + 'bootstrap': bootstrap, 'cronhub': cronhub, 'cronitor': cronitor, 'healthchecks': healthchecks, diff --git a/borgmatic/hooks/dump.py b/borgmatic/hooks/dump.py index ac380316..228c12bb 100644 --- a/borgmatic/hooks/dump.py +++ b/borgmatic/hooks/dump.py @@ -6,6 +6,7 @@ import shutil logger = logging.getLogger(__name__) DATA_SOURCE_HOOK_NAMES = ( + 'bootstrap', 'mariadb_databases', 'mysql_databases', 'mongodb_databases', diff --git a/borgmatic/hooks/mariadb.py b/borgmatic/hooks/mariadb.py index e7d829ef..f63d1515 100644 --- a/borgmatic/hooks/mariadb.py +++ b/borgmatic/hooks/mariadb.py @@ -126,6 +126,7 @@ def dump_data_sources( databases, config, log_prefix, + config_paths, borgmatic_runtime_directory, source_directories, dry_run, diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index 853711bd..2bb5c257 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -28,6 +28,7 @@ def dump_data_sources( databases, config, log_prefix, + config_paths, borgmatic_runtime_directory, source_directories, dry_run, diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index dfefcc3d..a085fc0b 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -125,6 +125,7 @@ def dump_data_sources( databases, config, log_prefix, + config_paths, borgmatic_runtime_directory, source_directories, dry_run, diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index eb0f296d..337c0f88 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -108,6 +108,7 @@ def dump_data_sources( databases, config, log_prefix, + config_paths, borgmatic_runtime_directory, source_directories, dry_run, diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py index fa0d0583..f14cb94b 100644 --- a/borgmatic/hooks/sqlite.py +++ b/borgmatic/hooks/sqlite.py @@ -28,6 +28,7 @@ def dump_data_sources( databases, config, log_prefix, + config_paths, borgmatic_runtime_directory, source_directories, dry_run, diff --git a/borgmatic/hooks/zfs.py b/borgmatic/hooks/zfs.py index 4fbcc785..6e000674 100644 --- a/borgmatic/hooks/zfs.py +++ b/borgmatic/hooks/zfs.py @@ -120,17 +120,19 @@ def dump_data_sources( hook_config, config, log_prefix, + config_paths, borgmatic_runtime_directory, source_directories, dry_run, ): ''' - Given a ZFS configuration dict, a configuration dict, a log prefix, the borgmatic runtime - directory, the configured source directories, and whether this is a dry run, auto-detect and - snapshot any ZFS dataset mount points listed in the given source directories and any dataset - with a borgmatic-specific user property. Also update those source directories, replacing dataset - mount points with corresponding snapshot directories so they get stored in the Borg archive - instead of the dataset mount points. Use the log prefix in any log entries. + Given a ZFS configuration dict, a configuration dict, a log prefix, the borgmatic configuration + file paths, the borgmatic runtime directory, the configured source directories, and whether this + is a dry run, auto-detect and snapshot any ZFS dataset mount points listed in the given source + directories and any dataset with a borgmatic-specific user property. Also update those source + directories, replacing dataset mount points with corresponding snapshot directories so they get + stored in the Borg archive instead of the dataset mount points. Use the log prefix in any log + entries. Return an empty sequence, since there are no ongoing dump processes from this hook. @@ -306,15 +308,24 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ destroy_snapshot(zfs_command, full_snapshot_name) -def make_data_source_dump_patterns(hook_config, config, log_prefix, name=None): # pragma: no cover +def make_data_source_dump_patterns( + hook_config, config, log_prefix, borgmatic_runtime_directory, name=None +): # pragma: no cover ''' Restores aren't implemented, because stored files can be extracted directly with "extract". ''' - raise NotImplementedError() + return () def restore_data_source_dump( - hook_config, config, log_prefix, data_source, dry_run, extract_process, connection_params + hook_config, + config, + log_prefix, + data_source, + dry_run, + extract_process, + connection_params, + borgmatic_runtime_directory, ): # pragma: no cover ''' Restores aren't implemented, because stored files can be extracted directly with "extract". diff --git a/docs/how-to/extract-a-backup.md b/docs/how-to/extract-a-backup.md index f6909110..7f5bb662 100644 --- a/docs/how-to/extract-a-backup.md +++ b/docs/how-to/extract-a-backup.md @@ -202,13 +202,25 @@ borgmatic config bootstrap --repository repo.borg --archive host-2023-01-02T04:0 See the output of `config bootstrap --help` for additional flags you may need for bootstrapping. -New in version 1.8.1 Set the -`store_config_files` option to `false` to disable the automatic backup of -borgmatic configuration files, for instance if they contain sensitive -information you don't want to store even inside your encrypted backups. If you -do this though, the `config bootstrap` action will no longer work. +New in version 1.9.3 +If your borgmatic configuration files contain sensitive information you don't +want to store even inside your encrypted backups, you can disable the +automatic backup of the configuration files. To do this, set the +`store_config_files` option under the `bootstrap` hook to `false`. For +instance: -New in version 1.8.7 Included -configuration files are stored in each backup archive. This means that the -`config bootstrap` action not only extracts the top-level configuration files -but also the includes they depend upon. +```yaml +bootstrap: + store_config_files: false +``` + +If you do this though, the `config bootstrap` action will no longer work. + +In version 1.8.1 through 1.9.2 +The `store_config_files` option was at the global scope instead of under the +`bootstrap` hook. + +New in version 1.8.7 +Configuration file includes are stored in each backup archive. This means that +the `config bootstrap` action not only extracts the top-level configuration +files but also the includes they depend upon. diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index b0c8712f..9cd5c980 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -67,6 +67,7 @@ def test_parse_configuration_transforms_file_into_mapping(): 'keep_hourly': 24, 'keep_minutely': 60, 'checks': [{'name': 'repository'}, {'name': 'archives'}], + 'bootstrap': {}, } assert config_paths == {'/tmp/config.yaml'} assert logs == [] @@ -90,6 +91,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): assert config == { 'source_directories': [f'/home/{string.punctuation}'], 'repositories': [{'path': 'test.borg'}], + 'bootstrap': {}, } assert config_paths == {'/tmp/config.yaml'} assert logs == [] @@ -150,6 +152,7 @@ def test_parse_configuration_inlines_include_inside_deprecated_section(): 'repositories': [{'path': 'hostname.borg'}], 'keep_daily': 7, 'keep_hourly': 24, + 'bootstrap': {}, } assert config_paths == {'/tmp/include.yaml', '/tmp/config.yaml'} assert len(logs) == 1 @@ -185,6 +188,7 @@ def test_parse_configuration_merges_include(): 'repositories': [{'path': 'hostname.borg'}], 'keep_daily': 1, 'keep_hourly': 24, + 'bootstrap': {}, } assert config_paths == {'/tmp/include.yaml', '/tmp/config.yaml'} assert logs == [] @@ -248,6 +252,7 @@ def test_parse_configuration_applies_overrides(): 'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}], 'local_path': 'borg2', + 'bootstrap': {}, } assert config_paths == {'/tmp/config.yaml'} assert logs == [] @@ -274,6 +279,7 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in 'source_directories': ['/home'], 'repositories': [{'path': 'ssh://user@hostname/./repo'}], 'exclude_if_present': ['.nobackup'], + 'bootstrap': {}, } assert config_paths == {'/tmp/config.yaml'} assert logs diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 607c9ffe..5e164bb7 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -1,61 +1,9 @@ -import sys - import pytest from flexmock import flexmock from borgmatic.actions import create as module -def test_create_borgmatic_manifest_creates_manifest_file(): - flexmock(module.os.path).should_receive('join').with_args( - '/run/borgmatic', 'bootstrap', 'manifest.json' - ).and_return('/run/borgmatic/bootstrap/manifest.json') - flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.os).should_receive('makedirs').and_return(True) - - flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0') - flexmock(sys.modules['builtins']).should_receive('open').with_args( - '/run/borgmatic/bootstrap/manifest.json', 'w' - ).and_return( - flexmock( - __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None), - __exit__=lambda *args: None, - ) - ) - flexmock(module.json).should_receive('dump').and_return(True).once() - - module.create_borgmatic_manifest({}, 'test.yaml', '/run/borgmatic', False) - - -def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_runtime_directory(): - flexmock(module.os.path).should_receive('join').with_args( - '/run/borgmatic', 'bootstrap', 'manifest.json' - ).and_return('/run/borgmatic/bootstrap/manifest.json') - flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.os).should_receive('makedirs').and_return(True) - - flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0') - flexmock(sys.modules['builtins']).should_receive('open').with_args( - '/run/borgmatic/bootstrap/manifest.json', 'w' - ).and_return( - flexmock( - __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None), - __exit__=lambda *args: None, - ) - ) - flexmock(module.json).should_receive('dump').and_return(True).once() - - module.create_borgmatic_manifest( - {'borgmatic_runtime_directory': '/borgmatic'}, 'test.yaml', '/run/borgmatic', False - ) - - -def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run(): - flexmock(module.json).should_receive('dump').never() - - module.create_borgmatic_manifest({}, 'test.yaml', '/run/borgmatic', True) - - def test_expand_directory_with_basic_path_passes_it_through(): flexmock(module.os.path).should_receive('expanduser').and_return('foo') flexmock(module.glob).should_receive('glob').and_return([]) @@ -207,28 +155,7 @@ def test_pattern_root_directories_parses_roots_and_ignores_others(): ) == ['/root', '/baz'] -def test_process_source_directories_includes_source_directories_and_config_paths(): - flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( - '/working' - ) - flexmock(module).should_receive('deduplicate_directories').and_return( - ('foo', 'bar', 'test.yaml') - ) - flexmock(module).should_receive('map_directories_to_devices').and_return({}) - flexmock(module).should_receive('expand_directories').with_args( - ('foo', 'bar', 'test.yaml'), working_directory='/working' - ).and_return(()).once() - flexmock(module).should_receive('pattern_root_directories').and_return(()) - flexmock(module).should_receive('expand_directories').with_args( - (), working_directory='/working' - ).and_return(()) - - assert module.process_source_directories( - config={'source_directories': ['foo', 'bar']}, config_paths=('test.yaml',) - ) == ('foo', 'bar', 'test.yaml') - - -def test_process_source_directories_does_not_include_config_paths_when_store_config_files_is_false(): +def test_process_source_directories_includes_source_directories(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( '/working' ) @@ -243,8 +170,7 @@ def test_process_source_directories_does_not_include_config_paths_when_store_con ).and_return(()) assert module.process_source_directories( - config={'source_directories': ['foo', 'bar'], 'store_config_files': False}, - config_paths=('test.yaml',), + config={'source_directories': ['foo', 'bar']}, ) == ('foo', 'bar') @@ -264,7 +190,6 @@ def test_process_source_directories_prefers_source_directory_argument_to_config( assert module.process_source_directories( config={'source_directories': ['nope']}, - config_paths=('test.yaml',), source_directories=['foo', 'bar'], ) == ('foo', 'bar') @@ -276,7 +201,6 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): flexmock() ) flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() - flexmock(module).should_receive('create_borgmatic_manifest').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( @@ -310,47 +234,6 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): ) -def test_run_create_with_store_config_files_false_does_not_create_borgmatic_manifest(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() - flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( - flexmock() - ) - flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() - flexmock(module).should_receive('create_borgmatic_manifest').never() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) - flexmock(module.borgmatic.hooks.dispatch).should_receive( - 'call_hooks_even_if_unconfigured' - ).and_return({}) - flexmock(module).should_receive('process_source_directories').and_return([]) - flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') - create_arguments = flexmock( - repository=None, - progress=flexmock(), - stats=flexmock(), - json=False, - list_files=flexmock(), - ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - list( - module.run_create( - config_filename='test.yaml', - repository={'path': 'repo'}, - config={'store_config_files': False}, - config_paths=['/tmp/test.yaml'], - hook_context={}, - local_borg_version=None, - create_arguments=create_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - ) - - def test_run_create_runs_with_selected_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( @@ -360,7 +243,6 @@ def test_run_create_runs_with_selected_repository(): flexmock() ) flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() - flexmock(module).should_receive('create_borgmatic_manifest').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( @@ -401,7 +283,6 @@ def test_run_create_bails_if_repository_does_not_match(): ).once().and_return(False) flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never() flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() - flexmock(module).should_receive('create_borgmatic_manifest').never() create_arguments = flexmock( repository=flexmock(), progress=flexmock(), @@ -441,7 +322,6 @@ def test_run_create_produces_json(): ) parsed_json = flexmock() flexmock(module.borgmatic.actions.json).should_receive('parse_json').and_return(parsed_json) - flexmock(module).should_receive('create_borgmatic_manifest').once() flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index 83f65a3d..edef6695 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -136,6 +136,11 @@ def test_normalize_sections_with_only_scalar_raises(): {'exclude_if_present': ['.nobackup']}, False, ), + ( + {'store_config_files': False}, + {'bootstrap': {'store_config_files': False}}, + True, + ), ( {'source_directories': ['foo', 'bar']}, {'source_directories': ['foo', 'bar']}, @@ -259,6 +264,7 @@ def test_normalize_applies_hard_coded_normalization_to_config( flexmock(module).should_receive('normalize_sections').and_return([]) logs = module.normalize('test.yaml', config) + expected_config.setdefault('bootstrap', {}) assert config == expected_config diff --git a/tests/unit/hooks/test_mariadb.py b/tests/unit/hooks/test_mariadb.py index 97db8ff2..3d6879fa 100644 --- a/tests/unit/hooks/test_mariadb.py +++ b/tests/unit/hooks/test_mariadb.py @@ -78,6 +78,7 @@ def test_dump_data_sources_dumps_each_database(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -108,6 +109,7 @@ def test_dump_data_sources_dumps_with_password(): [database], {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -133,6 +135,7 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -161,6 +164,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -472,6 +476,7 @@ def test_dump_data_sources_errors_for_missing_all_databases(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -491,6 +496,7 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=True, diff --git a/tests/unit/hooks/test_mongodb.py b/tests/unit/hooks/test_mongodb.py index 37e05d05..0a1cc0e0 100644 --- a/tests/unit/hooks/test_mongodb.py +++ b/tests/unit/hooks/test_mongodb.py @@ -46,6 +46,7 @@ def test_dump_data_sources_runs_mongodump_for_each_database(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -68,6 +69,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=True, @@ -106,6 +108,7 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -151,6 +154,7 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -176,6 +180,7 @@ def test_dump_data_sources_runs_mongodump_with_directory_format(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -211,6 +216,7 @@ def test_dump_data_sources_runs_mongodump_with_options(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -236,6 +242,7 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 2213c6c9..6efdc6f3 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -78,6 +78,7 @@ def test_dump_data_sources_dumps_each_database(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -108,6 +109,7 @@ def test_dump_data_sources_dumps_with_password(): [database], {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -133,6 +135,7 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -161,6 +164,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -470,6 +474,7 @@ def test_dump_data_sources_errors_for_missing_all_databases(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -489,6 +494,7 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=True, diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index 1026f066..2f32913f 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -256,6 +256,7 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -275,6 +276,7 @@ def test_dump_data_sources_raises_when_no_database_names_to_dump(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -291,6 +293,7 @@ def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=True, @@ -316,6 +319,7 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -343,6 +347,7 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=True, @@ -388,6 +393,7 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -431,6 +437,7 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -474,6 +481,7 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -513,6 +521,7 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -555,6 +564,7 @@ def test_dump_data_sources_runs_pg_dump_with_options(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -584,6 +594,7 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -625,6 +636,7 @@ def test_dump_data_sources_runs_non_default_pg_dump(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, diff --git a/tests/unit/hooks/test_sqlite.py b/tests/unit/hooks/test_sqlite.py index 96ece9b6..fa6f24e6 100644 --- a/tests/unit/hooks/test_sqlite.py +++ b/tests/unit/hooks/test_sqlite.py @@ -31,6 +31,7 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -61,6 +62,7 @@ def test_dump_data_sources_dumps_each_database(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -98,6 +100,7 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -126,6 +129,7 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -156,6 +160,7 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=False, @@ -180,6 +185,7 @@ def test_dump_data_sources_does_not_dump_if_dry_run(): databases, {}, 'test.yaml', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=[], dry_run=True, diff --git a/tests/unit/hooks/test_zfs.py b/tests/unit/hooks/test_zfs.py index 680e5166..e88d358c 100644 --- a/tests/unit/hooks/test_zfs.py +++ b/tests/unit/hooks/test_zfs.py @@ -77,6 +77,7 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_source_directories() hook_config={}, config={'source_directories': '/mnt/dataset', 'zfs': {}}, log_prefix='test', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=source_directories, dry_run=False, @@ -117,6 +118,7 @@ def test_dump_data_sources_uses_custom_commands(): 'zfs': hook_config, }, log_prefix='test', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=source_directories, dry_run=False, @@ -141,6 +143,7 @@ def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_source hook_config={}, config={'source_directories': '/mnt/dataset', 'zfs': {}}, log_prefix='test', + config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', source_directories=source_directories, dry_run=True,