From 814cdb4b870bc73c2b10949c0558fd60a4134860 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 3 Nov 2024 13:14:05 -0800 Subject: [PATCH] Deprecate the "borgmatic_source_directory" option in favor of "user_runtime_directory" and "user_state_directory" (#562). Move the default borgmatic streaming database dump and bootstrap metadata location on disk (#562). With Borg 1.4+, store database dumps and bootstrap metadata in a "/borgmatic" directory within a backup archive (#838). Add "--local-path", "--remote-path", and "--user-runtime-directory" flags to the "config bootstrap" action. --- NEWS | 20 +- borgmatic/actions/check.py | 3 +- borgmatic/actions/config/bootstrap.py | 94 ++++--- borgmatic/actions/create.py | 11 +- borgmatic/actions/restore.py | 139 +++++++--- borgmatic/borg/create.py | 31 +-- borgmatic/borg/extract.py | 7 +- borgmatic/borg/version.py | 2 +- borgmatic/commands/arguments.py | 20 +- borgmatic/commands/borgmatic.py | 4 +- borgmatic/config/paths.py | 30 ++- borgmatic/config/schema.yaml | 21 +- borgmatic/hooks/dump.py | 36 +-- borgmatic/hooks/mariadb.py | 23 +- borgmatic/hooks/mongodb.py | 23 +- borgmatic/hooks/mysql.py | 23 +- borgmatic/hooks/postgresql.py | 26 +- borgmatic/hooks/sqlite.py | 23 +- docs/how-to/backup-your-databases.md | 23 +- docs/how-to/deal-with-very-large-backups.md | 19 +- docs/how-to/inspect-your-backups.md | 24 +- docs/how-to/run-arbitrary-borg-commands.md | 2 +- tests/end-to-end/test_database.py | 29 +-- tests/unit/actions/config/test_bootstrap.py | 137 +++++++--- tests/unit/actions/test_check.py | 47 +++- tests/unit/actions/test_create.py | 18 +- tests/unit/actions/test_restore.py | 269 +++++++++++++++++++- tests/unit/borg/test_create.py | 154 ++++++----- tests/unit/commands/test_arguments.py | 9 +- tests/unit/commands/test_borgmatic.py | 6 +- tests/unit/config/test_paths.py | 26 +- tests/unit/hooks/test_dump.py | 24 +- 32 files changed, 952 insertions(+), 371 deletions(-) diff --git a/NEWS b/NEWS index 50fdb621..700c1cdc 100644 --- a/NEWS +++ b/NEWS @@ -3,12 +3,20 @@ option. * #609: BREAKING: Apply the "working_directory" option to all actions, not just "create". This includes repository paths, destination paths, mount points, etc. - * #562: Deprecate the "borgmatic_source_directory" option in favor of "borgmatic_runtime_directory" - and "borgmatic_state_directory". + * #562: Deprecate the "borgmatic_source_directory" option in favor of "user_runtime_directory" + and "user_state_directory". + * #562: BREAKING: Move the default borgmatic streaming database dump and bootstrap metadata + directory from ~/.borgmatic to /run/user/$UID/borgmatic, which is more XDG-compliant. Override + this location with the new "user_runtime_directory" option. Existing archives with database dumps + at the old location are still restorable. * #562, #638: Move the default check state directory from ~/.borgmatic to ~/.local/state/borgmatic. This is more XDG-compliant and also prevents these state files from - getting backed up (unless you include them). Override this location with the new - "borgmatic_state_directory" option. + getting backed up (unless you explicitly include them). Override this location with the new + "user_state_directory" option. After the first time you run the "check" action with borgmatic + 1.9.0, you can safely delete the ~/.borgmatic directory. + * #838: With Borg 1.4+, store database dumps and bootstrap metadata in a "/borgmatic" directory + within a backup archive, so the path doesn't depend on the current user. This means that you can + now backup as one user and restore or bootstrap as another user, among other use cases. * #902: Add loading of encrypted systemd credentials. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/#using-systemd-service-credentials * #914: Fix a confusing apparent hang when when the repository location changes, and instead @@ -36,6 +44,10 @@ * Update the "--match-archives" and "--archive" flags to support Borg 2 series names or archive hashes. * Add a "--match-archives" flag to the "prune" action. + * Add "--local-path" and "--remote-path" flags to the "config bootstrap" action for setting the + Borg executable paths used for bootstrapping. + * Add a "--user-runtime-directory" flag to the "config bootstrap" action for helping borgmatic + locate the bootstrap metadata stored in an archive. * Add a Zabbix monitoring hook. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook * Add a tarball of borgmatic's HTML documentation to the packages on the project page. diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index b8c71012..946111bd 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -368,7 +368,7 @@ def collect_spot_check_source_paths( config_paths=(), local_borg_version=local_borg_version, global_arguments=global_arguments, - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), local_path=local_path, remote_path=remote_path, list_files=True, @@ -427,6 +427,7 @@ def collect_spot_check_archive_paths( ) for (file_type, path) in (line.split(' ', 1),) if file_type != BORG_DIRECTORY_FILE_TYPE + if pathlib.Path('/borgmatic') not in pathlib.Path(path).parents if pathlib.Path(borgmatic_source_directory) not in pathlib.Path(path).parents if pathlib.Path(borgmatic_runtime_directory) not in pathlib.Path(path).parents ) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index ea3b8b41..bd1c6bde 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -4,51 +4,68 @@ import os import borgmatic.borg.extract import borgmatic.borg.repo_list +import borgmatic.config.paths import borgmatic.config.validate import borgmatic.hooks.command -from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY logger = logging.getLogger(__name__) -def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version): +def make_bootstrap_config(bootstrap_arguments): ''' - Given the bootstrap arguments as an argparse.Namespace (containing the repository and archive - name, borgmatic source directory, destination directory, and whether to strip components), the - global arguments as an argparse.Namespace (containing the dry run flag and the local borg - version), return the config paths from the manifest.json file in the borgmatic source directory - after extracting it from the repository. + Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict. + ''' + return { + 'ssh_command': bootstrap_arguments.ssh_command, + # In case the repo has been moved or is accessed from a different path at the point of + # bootstrapping. + 'relocated_repo_access_is_ok': True, + } + + +def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version): + ''' + Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the + repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory, + borgmatic source directory, destination directory, and whether to strip components), the global + arguments as an argparse.Namespace (containing the dry run flag and the local borg version), + return the config paths from the manifest.json file in the borgmatic source directory or runtime + directory after extracting it from the repository archive. Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the expected configuration path data. ''' - borgmatic_source_directory = ( - bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY + borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory( + {'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory} ) - borgmatic_manifest_path = os.path.expanduser( - os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json') + borgmatic_runtime_directory = borgmatic.config.paths.get_borgmatic_runtime_directory( + {'user_runtime_directory': bootstrap_arguments.user_runtime_directory} ) - config = {'ssh_command': bootstrap_arguments.ssh_command} + config = make_bootstrap_config(bootstrap_arguments) - extract_process = borgmatic.borg.extract.extract_archive( - global_arguments.dry_run, - bootstrap_arguments.repository, - borgmatic.borg.repo_list.resolve_archive_name( + # Probe for the manifest file in multiple locations, as the default location has moved to the + # borgmatic runtime directory (which get stored as just "/borgmatic" with Borg 1.4+). But we + # still want to support reading the manifest from previously created archives as well. + for base_directory in ('borgmatic', borgmatic_runtime_directory, borgmatic_source_directory): + borgmatic_manifest_path = os.path.join(base_directory, 'bootstrap', 'manifest.json') + + extract_process = borgmatic.borg.extract.extract_archive( + global_arguments.dry_run, bootstrap_arguments.repository, - bootstrap_arguments.archive, + archive_name, + [borgmatic_manifest_path], config, local_borg_version, global_arguments, - ), - [borgmatic_manifest_path], - config, - local_borg_version, - global_arguments, - extract_to_stdout=True, - ) - manifest_json = extract_process.stdout.read() + local_path=bootstrap_arguments.local_path, + remote_path=bootstrap_arguments.remote_path, + extract_to_stdout=True, + ) + manifest_json = extract_process.stdout.read() - if not manifest_json: + if manifest_json: + break + else: raise ValueError( 'Cannot read configuration paths from archive due to missing bootstrap manifest' ) @@ -75,27 +92,32 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): Raise ValueError if the bootstrap configuration could not be loaded. Raise CalledProcessError or OSError if Borg could not be run. ''' - manifest_config_paths = get_config_paths( - bootstrap_arguments, global_arguments, local_borg_version + config = make_bootstrap_config(bootstrap_arguments) + archive_name = borgmatic.borg.repo_list.resolve_archive_name( + bootstrap_arguments.repository, + bootstrap_arguments.archive, + config, + local_borg_version, + global_arguments, + local_path=bootstrap_arguments.local_path, + remote_path=bootstrap_arguments.remote_path, + ) + manifest_config_paths = get_config_paths( + archive_name, bootstrap_arguments, global_arguments, local_borg_version ) - config = {'ssh_command': bootstrap_arguments.ssh_command} logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}") borgmatic.borg.extract.extract_archive( global_arguments.dry_run, bootstrap_arguments.repository, - borgmatic.borg.repo_list.resolve_archive_name( - bootstrap_arguments.repository, - bootstrap_arguments.archive, - config, - local_borg_version, - global_arguments, - ), + archive_name, [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths], config, local_borg_version, global_arguments, + local_path=bootstrap_arguments.local_path, + remote_path=bootstrap_arguments.remote_path, extract_to_stdout=False, destination_path=bootstrap_arguments.destination, strip_components=bootstrap_arguments.strip_components, diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 6d15585e..aa8d6bc6 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -5,7 +5,7 @@ import os import borgmatic.actions.json import borgmatic.borg.create -import borgmatic.borg.state +import borgmatic.config.paths import borgmatic.config.validate import borgmatic.hooks.command import borgmatic.hooks.dispatch @@ -22,12 +22,9 @@ def create_borgmatic_manifest(config, config_paths, dry_run): if dry_run: return - borgmatic_source_directory = config.get( - 'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY - ) - - borgmatic_manifest_path = os.path.expanduser( - os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json') + borgmatic_runtime_directory = borgmatic.config.paths.get_borgmatic_runtime_directory(config) + borgmatic_manifest_path = os.path.join( + borgmatic_runtime_directory, 'bootstrap', 'manifest.json' ) if not os.path.exists(borgmatic_manifest_path): diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 2e57726c..3d8a873e 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -1,12 +1,15 @@ import copy import logging import os +import pathlib +import shutil +import tempfile import borgmatic.borg.extract import borgmatic.borg.list import borgmatic.borg.mount import borgmatic.borg.repo_list -import borgmatic.borg.state +import borgmatic.config.paths import borgmatic.config.validate import borgmatic.hooks.dispatch import borgmatic.hooks.dump @@ -61,6 +64,37 @@ def get_configured_data_source( ) +def strip_path_prefix_from_extracted_dump_destination( + destination_path, borgmatic_runtime_directory +): + ''' + Directory-format dump files get extracted into a temporary directory containing a path prefix + that depends how the files were stored in the archive. So, given the destination path where the + dump was extracted and the borgmatic runtime directory, move the dump files such that the + restore doesn't have to deal with that varying path prefix. + + For instance, if the dump was extracted to: + + /run/user/0/borgmatic/tmp1234/borgmatic/postgresql_databases/test/... + + or: + + /run/user/0/borgmatic/tmp1234/root/.borgmatic/postgresql_databases/test/... + + then this function moves it to: + + /run/user/0/borgmatic/postgresql_databases/test/... + ''' + for subdirectory_path, _, _ in os.walk(destination_path): + databases_directory = os.path.basename(subdirectory_path) + + if not databases_directory.endswith('_databases'): + continue + + os.rename(subdirectory_path, os.path.join(borgmatic_runtime_directory, databases_directory)) + break + + def restore_single_data_source( repository, config, @@ -72,7 +106,7 @@ def restore_single_data_source( hook_name, data_source, connection_params, -): # pragma: no cover +): ''' Given (among other things) an archive name, a data source hook name, the hostname, port, username/password as connection params, and a configured data source configuration dict, restore @@ -82,31 +116,48 @@ def restore_single_data_source( f'{repository.get("label", repository["path"])}: Restoring data source {data_source["name"]}' ) - dump_pattern = borgmatic.hooks.dispatch.call_hooks( - 'make_data_source_dump_pattern', + dump_patterns = borgmatic.hooks.dispatch.call_hooks( + 'make_data_source_dump_patterns', config, repository['path'], borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES, data_source['name'], )[hook_name] + borgmatic_runtime_directory = borgmatic.config.paths.get_borgmatic_runtime_directory(config) - # Kick off a single data source extract to stdout. - extract_process = borgmatic.borg.extract.extract_archive( - dry_run=global_arguments.dry_run, - repository=repository['path'], - archive=archive_name, - paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]), - config=config, - local_borg_version=local_borg_version, - global_arguments=global_arguments, - local_path=local_path, - remote_path=remote_path, - destination_path='/', - # A directory format dump isn't a single file, and therefore can't extract - # to stdout. In this case, the extract_process return value is None. - extract_to_stdout=bool(data_source.get('format') != 'directory'), + destination_path = ( + tempfile.mkdtemp(dir=borgmatic_runtime_directory) + if data_source.get('format') == 'directory' + else None ) + try: + # Kick off a single data source extract. If using a directory format, extract to a temporary + # directory. Otheriwes extract the single dump file to stdout. + extract_process = borgmatic.borg.extract.extract_archive( + dry_run=global_arguments.dry_run, + repository=repository['path'], + archive=archive_name, + paths=[borgmatic.hooks.dump.convert_glob_patterns_to_borg_pattern(dump_patterns)], + config=config, + local_borg_version=local_borg_version, + global_arguments=global_arguments, + local_path=local_path, + remote_path=remote_path, + destination_path=destination_path, + # A directory format dump isn't a single file, and therefore can't extract + # to stdout. In this case, the extract_process return value is None. + extract_to_stdout=bool(data_source.get('format') != 'directory'), + ) + + if destination_path and not global_arguments.dry_run: + strip_path_prefix_from_extracted_dump_destination( + destination_path, borgmatic_runtime_directory + ) + finally: + if destination_path and not global_arguments.dry_run: + shutil.rmtree(destination_path, ignore_errors=True) + # Run a single data source restore, consuming the extract stdout (if any). borgmatic.hooks.dispatch.call_hooks( function_name='restore_data_source_dump', @@ -135,11 +186,14 @@ def collect_archive_data_source_names( query the archive for the names of data sources it contains as dumps and return them as a dict from hook name to a sequence of data source names. ''' - borgmatic_source_directory = os.path.expanduser( - config.get( - 'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY - ) - ).lstrip('/') + borgmatic_source_directory = str( + pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config)) + ) + borgmatic_runtime_directory = borgmatic.config.paths.get_borgmatic_runtime_directory(config) + + # Probe for the data source dumps in multiple locations, as the default location has moved to + # the borgmatic runtime directory (which get stored as just "/borgmatic" with Borg 1.4+). But we + # still want to support reading dumps from previously created archives as well. dump_paths = borgmatic.borg.list.capture_archive_listing( repository, archive, @@ -148,10 +202,12 @@ def collect_archive_data_source_names( global_arguments, list_paths=[ 'sh:' - + os.path.expanduser( - borgmatic.hooks.dump.make_data_source_dump_path(borgmatic_source_directory, pattern) + + borgmatic.hooks.dump.make_data_source_dump_path(base_directory, '*_databases/*/*') + for base_directory in ( + 'borgmatic', + borgmatic_runtime_directory.lstrip('/'), + borgmatic_source_directory.lstrip('/'), ) - for pattern in ('*_databases/*/*',) ], local_path=local_path, remote_path=remote_path, @@ -162,17 +218,28 @@ def collect_archive_data_source_names( archive_data_source_names = {} for dump_path in dump_paths: - try: - (hook_name, _, data_source_name) = dump_path.split( - borgmatic_source_directory + os.path.sep, 1 - )[1].split(os.path.sep)[0:3] - except (ValueError, IndexError): + if not dump_path: + continue + + for base_directory in ( + 'borgmatic', + borgmatic_runtime_directory, + borgmatic_source_directory, + ): + try: + (hook_name, _, data_source_name) = dump_path.split(base_directory + os.path.sep, 1)[ + 1 + ].split(os.path.sep)[0:3] + except (ValueError, IndexError): + pass + else: + if data_source_name not in archive_data_source_names.get(hook_name, []): + archive_data_source_names.setdefault(hook_name, []).extend([data_source_name]) + break + else: logger.warning( f'{repository}: Ignoring invalid data source dump path "{dump_path}" in archive {archive}' ) - else: - if data_source_name not in archive_data_source_names.get(hook_name, []): - archive_data_source_names.setdefault(hook_name, []).extend([data_source_name]) return archive_data_source_names @@ -243,7 +310,7 @@ def ensure_data_sources_found(restore_names, remaining_restore_names, found_name ) if not combined_restore_names and not found_names: - raise ValueError('No data sources were found to restore') + raise ValueError('No data source dumps were found to restore') missing_names = sorted(set(combined_restore_names) - set(found_names)) if missing_names: diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index c04420e4..c34f7dfd 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -9,7 +9,7 @@ import textwrap import borgmatic.config.paths import borgmatic.logger -from borgmatic.borg import environment, feature, flags, state +from borgmatic.borg import environment, feature, flags from borgmatic.execute import ( DO_NOT_CAPTURE, execute_command, @@ -221,18 +221,13 @@ def make_list_filter_flags(local_borg_version, dry_run): return f'{base_flags}-' -def collect_borgmatic_source_directories(borgmatic_source_directory): +def collect_borgmatic_runtime_directories(borgmatic_runtime_directory): ''' - Return a list of borgmatic-specific source directories used for state like database backups. + Return a list of borgmatic-specific runtime directories used for temporary runtime data like + streaming database dumps and bootstrap metadata. If no such directories exist, return an empty + list. ''' - if not borgmatic_source_directory: - borgmatic_source_directory = state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY - - return ( - [borgmatic_source_directory] - if os.path.exists(os.path.expanduser(borgmatic_source_directory)) - else [] - ) + return [borgmatic_runtime_directory] if os.path.exists(borgmatic_runtime_directory) else [] ROOT_PATTERN_PREFIX = 'R ' @@ -342,7 +337,7 @@ def make_base_create_command( config_paths, local_borg_version, global_arguments, - borgmatic_source_directories, + borgmatic_runtime_directories, local_path='borg', remote_path=None, progress=False, @@ -368,7 +363,7 @@ def make_base_create_command( map_directories_to_devices( expand_directories( tuple(config.get('source_directories', ())) - + borgmatic_source_directories + + borgmatic_runtime_directories + tuple(config_paths if config.get('store_config_files', True) else ()), working_directory=working_directory, ) @@ -479,7 +474,7 @@ def make_base_create_command( local_path, working_directory, borg_environment, - skip_directories=borgmatic_source_directories, + skip_directories=borgmatic_runtime_directories, ) if special_file_paths: @@ -528,8 +523,10 @@ def create_archive( borgmatic.logger.add_custom_log_levels() working_directory = borgmatic.config.paths.get_working_directory(config) - borgmatic_source_directories = expand_directories( - collect_borgmatic_source_directories(config.get('borgmatic_source_directory')), + borgmatic_runtime_directories = expand_directories( + collect_borgmatic_runtime_directories( + borgmatic.config.paths.get_borgmatic_runtime_directory(config) + ), working_directory=working_directory, ) @@ -541,7 +538,7 @@ def create_archive( config_paths, local_borg_version, global_arguments, - borgmatic_source_directories, + borgmatic_runtime_directories, local_path, remote_path, progress, diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 5df0b868..67a6a63e 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -132,10 +132,9 @@ def extract_archive( + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( - # Make the repository path absolute so the destination directory - # used below via changing the working directory doesn't prevent - # Borg from finding the repo. But also apply the user's configured - # working directory (if any) to the repo path. + # Make the repository path absolute so the destination directory used below via changing + # the working directory doesn't prevent Borg from finding the repo. But also apply the + # user's configured working directory (if any) to the repo path. borgmatic.config.validate.normalize_repository_path( os.path.join(working_directory or '', repository) ), diff --git a/borgmatic/borg/version.py b/borgmatic/borg/version.py index 3156c375..e9cc4035 100644 --- a/borgmatic/borg/version.py +++ b/borgmatic/borg/version.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def local_borg_version(config, local_path='borg'): ''' - Given a configuration dict and a local Borg binary path, return a version string for it. + Given a configuration dict and a local Borg executable path, return a version string for it. Raise OSError or CalledProcessError if there is a problem running Borg. Raise ValueError if the version cannot be parsed. diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 76531bab..2eba9ff7 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -74,11 +74,11 @@ def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments for action_name, parsed in parsed_arguments.items(): for value in vars(parsed).values(): if isinstance(value, str): - if value in ACTION_ALIASES.keys(): + if value in ACTION_ALIASES.keys() and value in remaining_arguments: remaining_arguments.remove(value) elif isinstance(value, list): for item in value: - if item in ACTION_ALIASES.keys(): + if item in ACTION_ALIASES.keys() and item in remaining_arguments: remaining_arguments.remove(item) return tuple(remaining_arguments) @@ -864,9 +864,23 @@ def make_parsers(): help='Path of repository to extract config files from, quoted globs supported', required=True, ) + config_bootstrap_group.add_argument( + '--local-path', + help='Alternate Borg local executable. Defaults to "borg"', + default='borg', + ) + config_bootstrap_group.add_argument( + '--remote-path', + help='Alternate Borg remote executable. Defaults to "borg"', + default='borg', + ) + config_bootstrap_group.add_argument( + '--user-runtime-directory', + help='Path used for temporary runtime data like bootstrap metadata. Defaults to $XDG_RUNTIME_DIR or /var/run/$UID', + ) config_bootstrap_group.add_argument( '--borgmatic-source-directory', - help='Path that stores the config files used to create an archive and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic', + help='Deprecated. Path formerly used for temporary runtime data like bootstrap metadata. Defaults to ~/.borgmatic', ) config_bootstrap_group.add_argument( '--archive', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index d7c2aad8..1952703a 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -694,7 +694,9 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par if 'bootstrap' in arguments: try: # No configuration file is needed for bootstrap. - local_borg_version = borg_version.local_borg_version({}, 'borg') + local_borg_version = borg_version.local_borg_version( + {}, arguments['bootstrap'].local_path + ) except (OSError, CalledProcessError, ValueError) as error: yield from log_error_records('Error getting local Borg version', error) return diff --git a/borgmatic/config/paths.py b/borgmatic/config/paths.py index c971507c..d56ccc50 100644 --- a/borgmatic/config/paths.py +++ b/borgmatic/config/paths.py @@ -29,18 +29,21 @@ def get_borgmatic_source_directory(config): def get_borgmatic_runtime_directory(config): ''' Given a configuration dict, get the borgmatic runtime directory used for storing temporary - runtime data like streaming database dumps and bootstrap metadata. Defaults to the - "borgmatic_source_directory" value (deprecated) or $XDG_RUNTIME_DIR/borgmatic or - /var/run/$UID/borgmatic. + runtime data like streaming database dumps and bootstrap metadata. Defaults to + $XDG_RUNTIME_DIR/./borgmatic or /run/user/$UID/./borgmatic. + + The "/./" is taking advantage of a Borg feature such that the part of the path before the "/./" + does not get stored in the file path within an archive. That way, the path of the runtime + directory can change without leaving database dumps within an archive inaccessible. ''' return expand_user_in_path( - config.get('borgmatic_runtime_directory') - or config.get('borgmatic_source_directory') - or os.path.join( - os.environ.get( + os.path.join( + config.get('user_runtime_directory') + or os.environ.get( 'XDG_RUNTIME_DIR', - f'/var/run/{os.getuid()}', + f'/run/user/{os.getuid()}', ), + '.', 'borgmatic', ) ) @@ -49,14 +52,13 @@ def get_borgmatic_runtime_directory(config): def get_borgmatic_state_directory(config): ''' Given a configuration dict, get the borgmatic state directory used for storing borgmatic state - files like records of when checks last ran. Defaults to the "borgmatic_source_directory" value - (deprecated) or $XDG_STATE_HOME/borgmatic or ~/.local/state/borgmatic. + files like records of when checks last ran. Defaults to $XDG_STATE_HOME/borgmatic or + ~/.local/state/./borgmatic. ''' return expand_user_in_path( - config.get('borgmatic_state_directory') - or config.get('borgmatic_source_directory') - or os.path.join( - os.environ.get( + os.path.join( + config.get('user_state_directory') + or os.environ.get( 'XDG_STATE_HOME', '~/.local/state', ), diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3cf61a11..66ee665d 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -207,24 +207,27 @@ properties: borgmatic_source_directory: type: string description: | - Deprecated. Replaced by borgmatic_runtime_directory and + Deprecated. Only used for locating database dumps and bootstrap + metadata within backup archives created prior to deprecation. + Replaced by borgmatic_runtime_directory and borgmatic_state_directory. Defaults to ~/.borgmatic example: /tmp/borgmatic - borgmatic_runtime_directory: + user_runtime_directory: type: string description: | Path for storing temporary runtime data like streaming database - dumps and bootstrap metadata. Defaults to the - borgmatic_source_directory value (deprecated) or - $XDG_RUNTIME_DIR/borgmatic or /var/run/$UID/borgmatic. + dumps and bootstrap metadata. borgmatic automatically creates and + uses a "borgmatic" subdirectory here. Defaults to $XDG_RUNTIME_DIR + or /run/user/$UID. example: /run/user/1001/borgmatic - borgmatic_state_directory: + user_state_directory: type: string description: | Path for storing borgmatic state files like records of when checks - last ran. Defaults to the borgmatic_source_directory value - (deprecated) or $XDG_STATE_HOME/borgmatic or - ~/.local/state/borgmatic. + last ran. borgmatic automatically creates and uses a "borgmatic" + subdirectory here. If you change this option, borgmatic must + create the check records again (and therefore re-run checks). + Defaults to $XDG_STATE_HOME or ~/.local/state. example: /var/lib/borgmatic store_config_files: type: boolean diff --git a/borgmatic/hooks/dump.py b/borgmatic/hooks/dump.py index a04de74c..3082d926 100644 --- a/borgmatic/hooks/dump.py +++ b/borgmatic/hooks/dump.py @@ -1,9 +1,8 @@ +import fnmatch import logging import os import shutil -from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY - logger = logging.getLogger(__name__) DATA_SOURCE_HOOK_NAMES = ( @@ -15,15 +14,12 @@ DATA_SOURCE_HOOK_NAMES = ( ) -def make_data_source_dump_path(borgmatic_source_directory, data_source_hook_name): +def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_name): ''' - Given a borgmatic source directory (or None) and a data source hook name, construct a data - source dump path. + Given a borgmatic runtime directory and a data source hook name, construct a data source dump + path. ''' - if not borgmatic_source_directory: - borgmatic_source_directory = DEFAULT_BORGMATIC_SOURCE_DIRECTORY - - return os.path.join(borgmatic_source_directory, data_source_hook_name) + return os.path.join(borgmatic_runtime_directory, data_source_hook_name) def make_data_source_dump_filename(dump_path, name, hostname=None): @@ -36,7 +32,7 @@ def make_data_source_dump_filename(dump_path, name, hostname=None): if os.path.sep in name: raise ValueError(f'Invalid data source name {name}') - return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name) + return os.path.join(dump_path, hostname or 'localhost', name) def create_parent_directory_for_dump(dump_path): @@ -63,18 +59,22 @@ def remove_data_source_dumps(dump_path, data_source_type_name, log_prefix, dry_r logger.debug(f'{log_prefix}: Removing {data_source_type_name} data source dumps{dry_run_label}') - expanded_path = os.path.expanduser(dump_path) - if dry_run: return - if os.path.exists(expanded_path): - shutil.rmtree(expanded_path) + if os.path.exists(dump_path): + shutil.rmtree(dump_path) -def convert_glob_patterns_to_borg_patterns(patterns): +def convert_glob_patterns_to_borg_pattern(patterns): ''' - Convert a sequence of shell glob patterns like "/etc/*" to the corresponding Borg archive - patterns like "sh:etc/*". + Convert a sequence of shell glob patterns like "/etc/*", "/tmp/*" to the corresponding Borg + regular expression archive pattern as a single string like "re:etc/.*|tmp/.*". ''' - return [f'sh:{pattern.lstrip(os.path.sep)}' for pattern in patterns] + # Remove the "\Z" generated by fnmatch.translate() because we don't want the pattern to match + # only at the end of a path, as directory format dumps require extracting files with paths + # longer than the pattern. E.g., a pattern of "borgmatic/*/foo_databases/test" should also match + # paths like "borgmatic/*/foo_databases/test/toc.dat" + return 're:' + '|'.join( + fnmatch.translate(pattern.lstrip('/')).replace('\\Z', '') for pattern in patterns + ) diff --git a/borgmatic/hooks/mariadb.py b/borgmatic/hooks/mariadb.py index 9aa619fd..dcbd9d4b 100644 --- a/borgmatic/hooks/mariadb.py +++ b/borgmatic/hooks/mariadb.py @@ -3,6 +3,7 @@ import logging import os import shlex +import borgmatic.config.paths from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -13,12 +14,14 @@ from borgmatic.hooks import dump logger = logging.getLogger(__name__) -def make_dump_path(config): # pragma: no cover +def make_dump_path(config, base_directory=None): # pragma: no cover ''' - Make the dump path from the given configuration dict and the name of this hook. + Given a configuration dict and an optional base directory, make the corresponding dump path. If + a base directory isn't provided, use the borgmatic runtime directory. ''' return dump.make_data_source_dump_path( - config.get('borgmatic_source_directory'), 'mariadb_databases' + base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config), + 'mariadb_databases', ) @@ -191,13 +194,23 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: dump.remove_data_source_dumps(make_dump_path(config), 'MariaDB', log_prefix, dry_run) -def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover +def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover ''' Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a database name to match, return the corresponding glob patterns to match the database dump in an archive. ''' - return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*') + borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) + + return ( + dump.make_data_source_dump_filename( + make_dump_path(config, 'borgmatic'), name, hostname='*' + ), + dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'), + dump.make_data_source_dump_filename( + make_dump_path(config, borgmatic_source_directory), name, hostname='*' + ), + ) def restore_data_source_dump( diff --git a/borgmatic/hooks/mongodb.py b/borgmatic/hooks/mongodb.py index efb602fd..498b0638 100644 --- a/borgmatic/hooks/mongodb.py +++ b/borgmatic/hooks/mongodb.py @@ -1,18 +1,21 @@ import logging import shlex +import borgmatic.config.paths from borgmatic.execute import execute_command, execute_command_with_processes from borgmatic.hooks import dump logger = logging.getLogger(__name__) -def make_dump_path(config): # pragma: no cover +def make_dump_path(config, base_directory=None): # pragma: no cover ''' - Make the dump path from the given configuration dict and the name of this hook. + Given a configuration dict and an optional base directory, make the corresponding dump path. If + a base directory isn't provided, use the borgmatic runtime directory. ''' return dump.make_data_source_dump_path( - config.get('borgmatic_source_directory'), 'mongodb_databases' + base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config), + 'mongodb_databases', ) @@ -100,13 +103,23 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: dump.remove_data_source_dumps(make_dump_path(config), 'MongoDB', log_prefix, dry_run) -def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover +def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover ''' Given a sequence of database configurations dicts, a configuration dict, a prefix to log with, and a database name to match, return the corresponding glob patterns to match the database dump in an archive. ''' - return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*') + borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) + + return ( + dump.make_data_source_dump_filename( + make_dump_path(config, 'borgmatic'), name, hostname='*' + ), + dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'), + dump.make_data_source_dump_filename( + make_dump_path(config, borgmatic_source_directory), name, hostname='*' + ), + ) def restore_data_source_dump( diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index 8ffc778a..fb72da0f 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -3,6 +3,7 @@ import logging import os import shlex +import borgmatic.config.paths from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -13,12 +14,14 @@ from borgmatic.hooks import dump logger = logging.getLogger(__name__) -def make_dump_path(config): # pragma: no cover +def make_dump_path(config, base_directory=None): # pragma: no cover ''' - Make the dump path from the given configuration dict and the name of this hook. + Given a configuration dict and an optional base directory, make the corresponding dump path. If + a base directory isn't provided, use the borgmatic runtime directory. ''' return dump.make_data_source_dump_path( - config.get('borgmatic_source_directory'), 'mysql_databases' + base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config), + 'mysql_databases', ) @@ -189,13 +192,23 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: dump.remove_data_source_dumps(make_dump_path(config), 'MySQL', log_prefix, dry_run) -def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover +def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover ''' Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a database name to match, return the corresponding glob patterns to match the database dump in an archive. ''' - return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*') + borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) + + return ( + dump.make_data_source_dump_filename( + make_dump_path(config, 'borgmatic'), name, hostname='*' + ), + dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'), + dump.make_data_source_dump_filename( + make_dump_path(config, borgmatic_source_directory), name, hostname='*' + ), + ) def restore_data_source_dump( diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index 74b9f37e..1b510148 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -2,8 +2,10 @@ import csv import itertools import logging import os +import pathlib import shlex +import borgmatic.config.paths from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -14,12 +16,14 @@ from borgmatic.hooks import dump logger = logging.getLogger(__name__) -def make_dump_path(config): # pragma: no cover +def make_dump_path(config, base_directory=None): # pragma: no cover ''' - Make the dump path from the given configuration dict and the name of this hook. + Given a configuration dict and an optional base directory, make the corresponding dump path. If + a base directory isn't provided, use the borgmatic runtime directory. ''' return dump.make_data_source_dump_path( - config.get('borgmatic_source_directory'), 'postgresql_databases' + base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config), + 'postgresql_databases', ) @@ -215,13 +219,23 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: dump.remove_data_source_dumps(make_dump_path(config), 'PostgreSQL', log_prefix, dry_run) -def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover +def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover ''' Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a database name to match, return the corresponding glob patterns to match the database dump in an archive. ''' - return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*') + borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) + + return ( + dump.make_data_source_dump_filename( + make_dump_path(config, 'borgmatic'), name, hostname='*' + ), + dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'), + dump.make_data_source_dump_filename( + make_dump_path(config, borgmatic_source_directory), name, hostname='*' + ), + ) def restore_data_source_dump( @@ -291,7 +305,7 @@ def restore_data_source_dump( if 'restore_options' in data_source else () ) - + (() if extract_process else (dump_filename,)) + + (() if extract_process else (str(pathlib.Path(dump_filename)),)) + tuple( itertools.chain.from_iterable(('--schema', schema) for schema in data_source['schemas']) if data_source.get('schemas') diff --git a/borgmatic/hooks/sqlite.py b/borgmatic/hooks/sqlite.py index c7b494bf..778a6226 100644 --- a/borgmatic/hooks/sqlite.py +++ b/borgmatic/hooks/sqlite.py @@ -2,18 +2,21 @@ import logging import os import shlex +import borgmatic.config.paths from borgmatic.execute import execute_command, execute_command_with_processes from borgmatic.hooks import dump logger = logging.getLogger(__name__) -def make_dump_path(config): # pragma: no cover +def make_dump_path(config, base_directory=None): # pragma: no cover ''' - Make the dump path from the given configuration dict and the name of this hook. + Given a configuration dict and an optional base directory, make the corresponding dump path. If + a base directory isn't provided, use the borgmatic runtime directory. ''' return dump.make_data_source_dump_path( - config.get('borgmatic_source_directory'), 'sqlite_databases' + base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config), + 'sqlite_databases', ) @@ -87,12 +90,22 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: dump.remove_data_source_dumps(make_dump_path(config), 'SQLite', log_prefix, dry_run) -def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover +def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover ''' Make a pattern that matches the given SQLite databases. The databases are supplied as a sequence of configuration dicts, as per the configuration schema. ''' - return dump.make_data_source_dump_filename(make_dump_path(config), name) + borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) + + return ( + dump.make_data_source_dump_filename( + make_dump_path(config, 'borgmatic'), name, hostname='*' + ), + dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'), + dump.make_data_source_dump_filename( + make_dump_path(config, borgmatic_source_directory), name, hostname='*' + ), + ) def restore_data_source_dump( diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index fe9afd61..fc5aa5e7 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -63,14 +63,23 @@ dump formats, which can't stream and therefore do consume temporary disk space. Additionally, prior to borgmatic 1.5.3, all database dumps consumed temporary disk space.) -To support this, borgmatic creates temporary named pipes in `~/.borgmatic` by -default. To customize this path, set the `borgmatic_source_directory` option -in borgmatic's configuration. +New in version 1.9.0 To support +this, borgmatic creates temporary streaming database dumps within +`/run/user/$UID/borgmatic` by default (where `$UID` is the current user's ID). +To customize the `/run/user/$UID` portion of this path, set the +`user_runtime_directory` option in borgmatic's configuration. Alternatively, +set the `XDG_RUNTIME_DIR` environment variable (often already set to +`/run/user/$UID`). -Also note that using a database hook implicitly enables both the -`read_special` and `one_file_system` configuration settings (even if they're -disabled in your configuration) to support this dump and restore streaming. -See Limitations below for more on this. +Prior to version 1.9.0 +borgmatic created temporary streaming database dumps within the `~/.borgmatic` +directory by default. At that time, the path was configurable by the +`borgmatic_source_directory` configuration option (now deprecated). + +Also note that using a database hook implicitly enables the +`read_special` configuration option (even if it's disabled in your +configuration) to support this dump and restore streaming. See Limitations +below for more on this. Here's a more involved example that connects to remote databases: diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index 047ad28c..fbef5912 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -225,15 +225,26 @@ repository, at most once a month. Unlike a real scheduler like cron, borgmatic only makes a best effort to run checks on the configured frequency. It compares that frequency with how long -it's been since the last check for a given repository (as recorded in a file -within `~/.borgmatic/checks`). If it hasn't been long enough, the check is -skipped. And you still have to run `borgmatic check` (or `borgmatic` without -actions) in order for checks to run, even when a `frequency` is configured! +it's been since the last check for a given repository If it hasn't been long +enough, the check is skipped. And you still have to run `borgmatic check` (or +`borgmatic` without actions) in order for checks to run, even when a +`frequency` is configured! This also applies *across* configuration files that have the same repository configured. Make sure you have the same check frequency configured in each though—or the most frequently configured check will apply. +New in version 1.9.0To support +this frequency logic, borgmatic records check timestamps within the +`~/.local/state/borgmatic/checks` directory. To override the `~/.local/state` +portion of this path, set the `user_state_directory` configuration option. +Alternatively, set the `XDG_STATE_HOME` environment variable. + +Prior to version 1.9.0 +borgmatic recorded check timestamps within the `~/.borgmatic` directory. At +that time, the path was configurable by the `borgmatic_source_directory` +configuration option (now deprecated). + If you want to temporarily ignore your configured frequencies, you can invoke `borgmatic check --force` to run checks unconditionally. diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index 8aff6a3a..8b3676db 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -62,9 +62,9 @@ for available values. (No borgmatic `list` or `info` actions? Upgrade borgmatic!) -New in borgmatic version 1.9.0 -There are also `repo-list` and `repo-info` actions for displaying repository -information with Borg 2.x: +New in version 1.9.0 There are +also `repo-list` and `repo-info` actions for displaying repository information +with Borg 2.x: ```bash borgmatic repo-list @@ -107,12 +107,28 @@ hooks](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/), you can list backed up database dumps via borgmatic. For example: ```bash -borgmatic list --archive latest --find .borgmatic/*_databases +borgmatic list --archive latest --find *borgmatic/*_databases ``` This gives you a listing of all database dump files contained in the latest archive, complete with file sizes. +New in borgmatic version +1.9.0Database dump files are stored at `/borgmatic` within a backup +archive, regardless of the user who performs the backup. (Note that Borg +doesn't store the leading `/`.) + +With Borg version 1.2 and +earlierDatabase dump files are stored at `/var/run/$UID/borgmatic` +(where `$UID` is the current user's ID) unless overridden by the +`user_runtime_directory` configuration option or the `XDG_STATE_HOME` +environment variable. + +Prior to borgmatic version +1.9.0Database dump files were instead stored at `~/.borgmatic` within +the backup archive (where `~` was expanded to the home directory of the user +who performed the backup). This applied with all versions of Borg. + ## Logging diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index 20dfa654..dd231112 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -22,7 +22,7 @@ following, all based on your borgmatic configuration files or command-line arguments: * configured repositories, running your Borg command once for each one - * local and remote Borg binary paths + * local and remote Borg executable paths * SSH settings and Borg environment variables * lock wait settings * verbosity diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index b5337a75..826b8a3a 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -14,7 +14,7 @@ def write_configuration( source_directory, config_path, repository_path, - borgmatic_source_directory, + user_runtime_directory, postgresql_dump_format='custom', mongodb_dump_format='archive', ): @@ -28,7 +28,7 @@ source_directories: - {source_directory} repositories: - path: {repository_path} -borgmatic_source_directory: {borgmatic_source_directory} +user_runtime_directory: {user_runtime_directory} encryption_passphrase: "test" @@ -101,7 +101,7 @@ def write_custom_restore_configuration( source_directory, config_path, repository_path, - borgmatic_source_directory, + user_runtime_directory, postgresql_dump_format='custom', mongodb_dump_format='archive', ): @@ -115,7 +115,7 @@ source_directories: - {source_directory} repositories: - path: {repository_path} -borgmatic_source_directory: {borgmatic_source_directory} +user_runtime_directory: {user_runtime_directory} encryption_passphrase: "test" @@ -173,7 +173,7 @@ def write_simple_custom_restore_configuration( source_directory, config_path, repository_path, - borgmatic_source_directory, + user_runtime_directory, postgresql_dump_format='custom', ): ''' @@ -187,7 +187,7 @@ source_directories: - {source_directory} repositories: - path: {repository_path} -borgmatic_source_directory: {borgmatic_source_directory} +user_runtime_directory: {user_runtime_directory} encryption_passphrase: "test" @@ -347,7 +347,6 @@ def test_database_dump_and_restore(): # Create a Borg repository. temporary_directory = tempfile.mkdtemp() repository_path = os.path.join(temporary_directory, 'test.borg') - borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic') # Write out a special file to ensure that it gets properly excluded and Borg doesn't hang on it. os.mkfifo(os.path.join(temporary_directory, 'special_file')) @@ -357,7 +356,7 @@ def test_database_dump_and_restore(): try: config_path = os.path.join(temporary_directory, 'test.yaml') config = write_configuration( - temporary_directory, config_path, repository_path, borgmatic_source_directory + temporary_directory, config_path, repository_path, temporary_directory ) create_test_tables(config) select_test_tables(config) @@ -406,14 +405,13 @@ def test_database_dump_and_restore_with_restore_cli_flags(): # Create a Borg repository. temporary_directory = tempfile.mkdtemp() repository_path = os.path.join(temporary_directory, 'test.borg') - borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic') original_working_directory = os.getcwd() try: config_path = os.path.join(temporary_directory, 'test.yaml') config = write_simple_custom_restore_configuration( - temporary_directory, config_path, repository_path, borgmatic_source_directory + temporary_directory, config_path, repository_path, temporary_directory ) create_test_tables(config) select_test_tables(config) @@ -485,14 +483,13 @@ def test_database_dump_and_restore_with_restore_configuration_options(): # Create a Borg repository. temporary_directory = tempfile.mkdtemp() repository_path = os.path.join(temporary_directory, 'test.borg') - borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic') original_working_directory = os.getcwd() try: config_path = os.path.join(temporary_directory, 'test.yaml') config = write_custom_restore_configuration( - temporary_directory, config_path, repository_path, borgmatic_source_directory + temporary_directory, config_path, repository_path, temporary_directory ) create_test_tables(config) select_test_tables(config) @@ -542,7 +539,6 @@ def test_database_dump_and_restore_with_directory_format(): # Create a Borg repository. temporary_directory = tempfile.mkdtemp() repository_path = os.path.join(temporary_directory, 'test.borg') - borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic') original_working_directory = os.getcwd() @@ -552,7 +548,7 @@ def test_database_dump_and_restore_with_directory_format(): temporary_directory, config_path, repository_path, - borgmatic_source_directory, + temporary_directory, postgresql_dump_format='directory', mongodb_dump_format='directory', ) @@ -593,15 +589,12 @@ def test_database_dump_with_error_causes_borgmatic_to_exit(): # Create a Borg repository. temporary_directory = tempfile.mkdtemp() repository_path = os.path.join(temporary_directory, 'test.borg') - borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic') original_working_directory = os.getcwd() try: config_path = os.path.join(temporary_directory, 'test.yaml') - write_configuration( - temporary_directory, config_path, repository_path, borgmatic_source_directory - ) + write_configuration(temporary_directory, config_path, repository_path, temporary_directory) subprocess.check_call( [ diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index 8d419ecd..dceabe19 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -4,12 +4,30 @@ from flexmock import flexmock from borgmatic.actions.config import bootstrap as module +def test_make_bootstrap_config_uses_ssh_command_argument(): + ssh_command = flexmock() + + config = module.make_bootstrap_config(flexmock(ssh_command=ssh_command)) + assert config['ssh_command'] == ssh_command + assert config['relocated_repo_access_is_ok'] + + def test_get_config_paths_returns_list_of_config_paths(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/source') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/runtime') + flexmock(module).should_receive('make_bootstrap_config').and_return({}) bootstrap_arguments = flexmock( - borgmatic_source_directory=None, repository='repo', archive='archive', ssh_command=None, + local_path='borg7', + remote_path='borg8', + borgmatic_source_directory=None, + user_runtime_directory=None, ) global_arguments = flexmock( dry_run=False, @@ -23,20 +41,29 @@ def test_get_config_paths_returns_list_of_config_paths(): flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( extract_process ) - flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return( - 'archive' - ) - assert module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) == [ - '/borgmatic/config.yaml' - ] + + assert module.get_config_paths( + 'archive', bootstrap_arguments, global_arguments, local_borg_version + ) == ['/borgmatic/config.yaml'] def test_get_config_paths_translates_ssh_command_argument_to_config(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/source') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/runtime') + config = flexmock() + flexmock(module).should_receive('make_bootstrap_config').and_return(config) bootstrap_arguments = flexmock( - borgmatic_source_directory=None, repository='repo', archive='archive', ssh_command='ssh -i key', + local_path='borg7', + remote_path='borg8', + borgmatic_source_directory=None, + user_runtime_directory=None, ) global_arguments = flexmock( dry_run=False, @@ -52,25 +79,35 @@ def test_get_config_paths_translates_ssh_command_argument_to_config(): 'repo', 'archive', object, - {'ssh_command': 'ssh -i key'}, + config, object, object, extract_to_stdout=True, + local_path='borg7', + remote_path='borg8', ).and_return(extract_process) - flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').with_args( - 'repo', 'archive', {'ssh_command': 'ssh -i key'}, object, object - ).and_return('archive') - assert module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) == [ - '/borgmatic/config.yaml' - ] + + assert module.get_config_paths( + 'archive', bootstrap_arguments, global_arguments, local_borg_version + ) == ['/borgmatic/config.yaml'] def test_get_config_paths_with_missing_manifest_raises_value_error(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/source') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/runtime') + flexmock(module).should_receive('make_bootstrap_config').and_return({}) bootstrap_arguments = flexmock( - borgmatic_source_directory=None, repository='repo', archive='archive', ssh_command=None, + local_path='borg7', + remote_path='borg7', + borgmatic_source_directory=None, + user_runtime_directory=None, ) global_arguments = flexmock( dry_run=False, @@ -80,20 +117,29 @@ def test_get_config_paths_with_missing_manifest_raises_value_error(): flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( extract_process ) - flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return( - 'archive' - ) with pytest.raises(ValueError): - module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) + module.get_config_paths( + 'archive', bootstrap_arguments, global_arguments, local_borg_version + ) def test_get_config_paths_with_broken_json_raises_value_error(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/source') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/runtime') + flexmock(module).should_receive('make_bootstrap_config').and_return({}) bootstrap_arguments = flexmock( - borgmatic_source_directory=None, repository='repo', archive='archive', ssh_command=None, + local_path='borg7', + remote_path='borg7', + borgmatic_source_directory=None, + user_runtime_directory=None, ) global_arguments = flexmock( dry_run=False, @@ -105,20 +151,29 @@ def test_get_config_paths_with_broken_json_raises_value_error(): flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( extract_process ) - flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return( - 'archive' - ) with pytest.raises(ValueError): - module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) + module.get_config_paths( + 'archive', bootstrap_arguments, global_arguments, local_borg_version + ) def test_get_config_paths_with_json_missing_key_raises_value_error(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/source') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/runtime') + flexmock(module).should_receive('make_bootstrap_config').and_return({}) bootstrap_arguments = flexmock( - borgmatic_source_directory=None, repository='repo', archive='archive', ssh_command=None, + local_path='borg7', + remote_path='borg7', + borgmatic_source_directory=None, + user_runtime_directory=None, ) global_arguments = flexmock( dry_run=False, @@ -130,15 +185,15 @@ def test_get_config_paths_with_json_missing_key_raises_value_error(): flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( extract_process ) - flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return( - 'archive' - ) with pytest.raises(ValueError): - module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) + module.get_config_paths( + 'archive', bootstrap_arguments, global_arguments, local_borg_version + ) def test_run_bootstrap_does_not_raise(): + flexmock(module).should_receive('make_bootstrap_config').and_return({}) flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml']) bootstrap_arguments = flexmock( repository='repo', @@ -146,8 +201,10 @@ def test_run_bootstrap_does_not_raise(): destination='dest', strip_components=1, progress=False, - borgmatic_source_directory='/borgmatic', + user_runtime_directory='/borgmatic', ssh_command=None, + local_path='borg7', + remote_path='borg8', ) global_arguments = flexmock( dry_run=False, @@ -169,6 +226,8 @@ def test_run_bootstrap_does_not_raise(): def test_run_bootstrap_translates_ssh_command_argument_to_config(): + config = flexmock() + flexmock(module).should_receive('make_bootstrap_config').and_return(config) flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml']) bootstrap_arguments = flexmock( repository='repo', @@ -176,8 +235,10 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): destination='dest', strip_components=1, progress=False, - borgmatic_source_directory='/borgmatic', + user_runtime_directory='/borgmatic', ssh_command='ssh -i key', + local_path='borg7', + remote_path='borg8', ) global_arguments = flexmock( dry_run=False, @@ -193,16 +254,24 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): 'repo', 'archive', object, - {'ssh_command': 'ssh -i key'}, + config, object, object, extract_to_stdout=False, destination_path='dest', strip_components=1, progress=False, + local_path='borg7', + remote_path='borg8', ).and_return(extract_process).once() flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').with_args( - 'repo', 'archive', {'ssh_command': 'ssh -i key'}, object, object + 'repo', + 'archive', + config, + object, + object, + local_path='borg7', + remote_path='borg8', ).and_return('archive') module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version) diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index d8893e06..05e9a6b0 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -564,7 +564,7 @@ def test_collect_spot_check_source_paths_parses_borg_output(): config_paths=(), local_borg_version=object, global_arguments=object, - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), local_path=object, remote_path=object, list_files=True, @@ -602,7 +602,7 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false() config_paths=(), local_borg_version=object, global_arguments=object, - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), local_path=object, remote_path=object, list_files=True, @@ -640,7 +640,7 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o config_paths=(), local_borg_version=object, global_arguments=object, - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), local_path=object, remote_path=object, list_files=True, @@ -678,7 +678,7 @@ def test_collect_spot_check_source_paths_skips_directories(): config_paths=(), local_borg_version=object, global_arguments=object, - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), local_path=object, remote_path=object, list_files=True, @@ -715,7 +715,7 @@ def test_collect_spot_check_archive_paths_excludes_directories(): ).and_return('/home/user/.borgmatic') flexmock(module.borgmatic.config.paths).should_receive( 'get_borgmatic_runtime_directory' - ).and_return('/var/run/1001/borgmatic') + ).and_return('/run/user/1001/borgmatic') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( ( 'f /etc/path', @@ -735,13 +735,38 @@ def test_collect_spot_check_archive_paths_excludes_directories(): ) == ('/etc/path', '/etc/other') +def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory_as_stored_with_prefix_truncation(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + ( + 'f /etc/path', + 'f /borgmatic/some/thing', + ) + ) + + assert module.collect_spot_check_archive_paths( + repository={'path': 'repo'}, + archive='archive', + config={}, + local_borg_version=flexmock(), + global_arguments=flexmock(), + local_path=flexmock(), + remote_path=flexmock(), + ) == ('/etc/path',) + + def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_directory(): flexmock(module.borgmatic.config.paths).should_receive( 'get_borgmatic_source_directory' ).and_return('/root/.borgmatic') flexmock(module.borgmatic.config.paths).should_receive( 'get_borgmatic_runtime_directory' - ).and_return('/var/run/0/borgmatic') + ).and_return('/run/user/0/borgmatic') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( ( 'f /etc/path', @@ -752,7 +777,7 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_dire assert module.collect_spot_check_archive_paths( repository={'path': 'repo'}, archive='archive', - config={'borgmatic_source_directory': '/root/.borgmatic'}, + config={}, local_borg_version=flexmock(), global_arguments=flexmock(), local_path=flexmock(), @@ -766,18 +791,18 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_dir ).and_return('/root.borgmatic') flexmock(module.borgmatic.config.paths).should_receive( 'get_borgmatic_runtime_directory' - ).and_return('/var/run/0/borgmatic') + ).and_return('/run/user/0/borgmatic') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( ( 'f /etc/path', - 'f /var/run/0/borgmatic/some/thing', + 'f /run/user/0/borgmatic/some/thing', ) ) assert module.collect_spot_check_archive_paths( repository={'path': 'repo'}, archive='archive', - config={'borgmatic_runtime_directory': '/var/run/0/borgmatic'}, + config={}, local_borg_version=flexmock(), global_arguments=flexmock(), local_path=flexmock(), @@ -796,7 +821,7 @@ def test_collect_spot_check_source_paths_uses_working_directory(): config_paths=(), local_borg_version=object, global_arguments=object, - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), local_path=object, remote_path=object, list_files=True, diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 824bd9ab..8582abae 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -191,15 +191,18 @@ def test_run_create_produces_json(): def test_create_borgmatic_manifest_creates_manifest_file(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') flexmock(module.os.path).should_receive('join').with_args( - module.borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY, 'bootstrap', 'manifest.json' - ).and_return('/home/user/.borgmatic/bootstrap/manifest.json') + '/run/user/0/borgmatic', 'bootstrap', 'manifest.json' + ).and_return('/run/user/0/borgmatic/bootstrap/manifest.json') flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.os).should_receive('makedirs').and_return(True) flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0') flexmock(sys.modules['builtins']).should_receive('open').with_args( - '/home/user/.borgmatic/bootstrap/manifest.json', 'w' + '/run/user/0/borgmatic/bootstrap/manifest.json', 'w' ).and_return( flexmock( __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None), @@ -211,7 +214,10 @@ def test_create_borgmatic_manifest_creates_manifest_file(): module.create_borgmatic_manifest({}, 'test.yaml', False) -def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_source_directory(): +def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_runtime_directory(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/borgmatic') flexmock(module.os.path).should_receive('join').with_args( '/borgmatic', 'bootstrap', 'manifest.json' ).and_return('/borgmatic/bootstrap/manifest.json') @@ -230,11 +236,9 @@ def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_s flexmock(module.json).should_receive('dump').and_return(True).once() module.create_borgmatic_manifest( - {'borgmatic_source_directory': '/borgmatic'}, 'test.yaml', False + {'borgmatic_runtime_directory': '/borgmatic'}, 'test.yaml', False ) def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run(): - flexmock(module.os.path).should_receive('expanduser').never() - module.create_borgmatic_manifest({}, 'test.yaml', True) diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 89a48d28..31393e0b 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -65,22 +65,219 @@ def test_get_configured_data_source_with_unspecified_hook_matches_data_source_by ) == ('postgresql_databases', {'name': 'bar'}) +def test_strip_path_prefix_from_extracted_dump_destination_renames_first_matching_databases_subdirectory(): + flexmock(module.os).should_receive('walk').and_return( + [ + ('/foo', flexmock(), flexmock()), + ('/foo/bar', flexmock(), flexmock()), + ('/foo/bar/postgresql_databases', flexmock(), flexmock()), + ('/foo/bar/mariadb_databases', flexmock(), flexmock()), + ] + ) + + flexmock(module.os).should_receive('rename').with_args( + '/foo/bar/postgresql_databases', '/run/user/0/borgmatic/postgresql_databases' + ).once() + flexmock(module.os).should_receive('rename').with_args( + '/foo/bar/mariadb_databases', '/run/user/0/borgmatic/mariadb_databases' + ).never() + + module.strip_path_prefix_from_extracted_dump_destination('/foo', '/run/user/0/borgmatic') + + +def test_restore_single_data_source_extracts_and_restores_single_file_dump(): + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( + 'make_data_source_dump_patterns', object, object, object, object + ).and_return({'postgresql': flexmock()}) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') + flexmock(module.tempfile).should_receive('mkdtemp').never() + flexmock(module.borgmatic.hooks.dump).should_receive( + 'convert_glob_patterns_to_borg_pattern' + ).and_return(flexmock()) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + flexmock() + ).once() + flexmock(module).should_receive('strip_path_prefix_from_extracted_dump_destination').never() + flexmock(module.shutil).should_receive('rmtree').never() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( + function_name='restore_data_source_dump', + config=object, + log_prefix=object, + hook_names=object, + data_source=object, + dry_run=object, + extract_process=object, + connection_params=object, + ).once() + + module.restore_single_data_source( + repository={'path': 'test.borg'}, + config=flexmock(), + local_borg_version=flexmock(), + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + archive_name=flexmock(), + hook_name='postgresql', + data_source={'name': 'test', 'format': 'plain'}, + connection_params=flexmock(), + ) + + +def test_restore_single_data_source_extracts_and_restores_directory_dump(): + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( + 'make_data_source_dump_patterns', object, object, object, object + ).and_return({'postgresql': flexmock()}) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') + flexmock(module.tempfile).should_receive('mkdtemp').once().and_return( + '/run/user/0/borgmatic/tmp1234' + ) + flexmock(module.borgmatic.hooks.dump).should_receive( + 'convert_glob_patterns_to_borg_pattern' + ).and_return(flexmock()) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + flexmock() + ).once() + flexmock(module).should_receive('strip_path_prefix_from_extracted_dump_destination').once() + flexmock(module.shutil).should_receive('rmtree').once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( + function_name='restore_data_source_dump', + config=object, + log_prefix=object, + hook_names=object, + data_source=object, + dry_run=object, + extract_process=object, + connection_params=object, + ).once() + + module.restore_single_data_source( + repository={'path': 'test.borg'}, + config=flexmock(), + local_borg_version=flexmock(), + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + archive_name=flexmock(), + hook_name='postgresql', + data_source={'name': 'test', 'format': 'directory'}, + connection_params=flexmock(), + ) + + +def test_restore_single_data_source_with_directory_dump_error_cleans_up_temporary_directory(): + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( + 'make_data_source_dump_patterns', object, object, object, object + ).and_return({'postgresql': flexmock()}) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') + flexmock(module.tempfile).should_receive('mkdtemp').once().and_return( + '/run/user/0/borgmatic/tmp1234' + ) + flexmock(module.borgmatic.hooks.dump).should_receive( + 'convert_glob_patterns_to_borg_pattern' + ).and_return(flexmock()) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_raise( + ValueError + ).once() + flexmock(module).should_receive('strip_path_prefix_from_extracted_dump_destination').never() + flexmock(module.shutil).should_receive('rmtree').once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( + function_name='restore_data_source_dump', + config=object, + log_prefix=object, + hook_names=object, + data_source=object, + dry_run=object, + extract_process=object, + connection_params=object, + ).never() + + with pytest.raises(ValueError): + module.restore_single_data_source( + repository={'path': 'test.borg'}, + config=flexmock(), + local_borg_version=flexmock(), + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + archive_name=flexmock(), + hook_name='postgresql', + data_source={'name': 'test', 'format': 'directory'}, + connection_params=flexmock(), + ) + + +def test_restore_single_data_source_with_directory_dump_and_dry_run_skips_directory_move_and_cleanup(): + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( + 'make_data_source_dump_patterns', object, object, object, object + ).and_return({'postgresql': flexmock()}) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') + flexmock(module.tempfile).should_receive('mkdtemp').once().and_return( + '/run/user/0/borgmatic/tmp1234' + ) + flexmock(module.borgmatic.hooks.dump).should_receive( + 'convert_glob_patterns_to_borg_pattern' + ).and_return(flexmock()) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + flexmock() + ).once() + flexmock(module).should_receive('strip_path_prefix_from_extracted_dump_destination').never() + flexmock(module.shutil).should_receive('rmtree').never() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( + function_name='restore_data_source_dump', + config=object, + log_prefix=object, + hook_names=object, + data_source=object, + dry_run=object, + extract_process=object, + connection_params=object, + ).once() + + module.restore_single_data_source( + repository={'path': 'test.borg'}, + config=flexmock(), + local_borg_version=flexmock(), + global_arguments=flexmock(dry_run=True), + local_path=None, + remote_path=None, + archive_name=flexmock(), + hook_name='postgresql', + data_source={'name': 'test', 'format': 'directory'}, + connection_params=flexmock(), + ) + + def test_collect_archive_data_source_names_parses_archive_paths(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') flexmock(module.borgmatic.hooks.dump).should_receive('make_data_source_dump_path').and_return( '' ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( [ - '.borgmatic/postgresql_databases/localhost/foo', - '.borgmatic/postgresql_databases/localhost/bar', - '.borgmatic/mysql_databases/localhost/quux', + 'borgmatic/postgresql_databases/localhost/foo', + 'borgmatic/postgresql_databases/localhost/bar', + 'borgmatic/mysql_databases/localhost/quux', ] ) archive_data_source_names = module.collect_archive_data_source_names( repository={'path': 'repo'}, archive='archive', - config={'borgmatic_source_directory': '.borgmatic'}, + config={}, local_borg_version=flexmock(), global_arguments=flexmock(log_json=False), local_path=flexmock(), @@ -93,21 +290,62 @@ def test_collect_archive_data_source_names_parses_archive_paths(): } -def test_collect_archive_data_source_names_parses_directory_format_archive_paths(): +def test_collect_archive_data_source_names_parses_archive_paths_with_different_base_directories(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') flexmock(module.borgmatic.hooks.dump).should_receive('make_data_source_dump_path').and_return( '' ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( [ - '.borgmatic/postgresql_databases/localhost/foo/table1', - '.borgmatic/postgresql_databases/localhost/foo/table2', + 'borgmatic/postgresql_databases/localhost/foo', + '.borgmatic/postgresql_databases/localhost/bar', + '/root/.borgmatic/postgresql_databases/localhost/baz', + '/var/run/0/borgmatic/mysql_databases/localhost/quux', ] ) archive_data_source_names = module.collect_archive_data_source_names( repository={'path': 'repo'}, archive='archive', - config={'borgmatic_source_directory': '.borgmatic'}, + config={}, + local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), + local_path=flexmock(), + remote_path=flexmock(), + ) + + assert archive_data_source_names == { + 'postgresql_databases': ['foo', 'bar', 'baz'], + 'mysql_databases': ['quux'], + } + + +def test_collect_archive_data_source_names_parses_directory_format_archive_paths(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') + flexmock(module.borgmatic.hooks.dump).should_receive('make_data_source_dump_path').and_return( + '' + ) + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + [ + 'borgmatic/postgresql_databases/localhost/foo/table1', + 'borgmatic/postgresql_databases/localhost/foo/table2', + ] + ) + + archive_data_source_names = module.collect_archive_data_source_names( + repository={'path': 'repo'}, + archive='archive', + config={}, local_borg_version=flexmock(), global_arguments=flexmock(log_json=False), local_path=flexmock(), @@ -120,17 +358,28 @@ def test_collect_archive_data_source_names_parses_directory_format_archive_paths def test_collect_archive_data_source_names_skips_bad_archive_paths(): + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory' + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/run/user/0/borgmatic') flexmock(module.borgmatic.hooks.dump).should_receive('make_data_source_dump_path').and_return( '' ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( - ['.borgmatic/postgresql_databases/localhost/foo', '.borgmatic/invalid', 'invalid/as/well'] + [ + 'borgmatic/postgresql_databases/localhost/foo', + 'borgmatic/invalid', + 'invalid/as/well', + '', + ] ) archive_data_source_names = module.collect_archive_data_source_names( repository={'path': 'repo'}, archive='archive', - config={'borgmatic_source_directory': '.borgmatic'}, + config={}, local_borg_version=flexmock(), global_arguments=flexmock(log_json=False), local_path=flexmock(), diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index e64a20be..f24707a5 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -372,27 +372,16 @@ def test_make_list_filter_flags_with_info_and_feature_not_available_omits_x(): assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME-' -def test_collect_borgmatic_source_directories_set_when_directory_exists(): +def test_collect_borgmatic_runtime_directories_set_when_directory_exists(): flexmock(module.os.path).should_receive('exists').and_return(True) - flexmock(module.os.path).should_receive('expanduser') - assert module.collect_borgmatic_source_directories('/tmp') == ['/tmp'] + assert module.collect_borgmatic_runtime_directories('/tmp') == ['/tmp'] -def test_collect_borgmatic_source_directories_empty_when_directory_does_not_exist(): +def test_collect_borgmatic_runtime_directories_empty_when_directory_does_not_exist(): flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.os.path).should_receive('expanduser') - assert module.collect_borgmatic_source_directories('/tmp') == [] - - -def test_collect_borgmatic_source_directories_defaults_when_directory_not_given(): - flexmock(module.os.path).should_receive('exists').and_return(True) - flexmock(module.os.path).should_receive('expanduser') - - assert module.collect_borgmatic_source_directories(None) == [ - module.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY - ] + assert module.collect_borgmatic_runtime_directories('/tmp') == [] def test_pattern_root_directories_deals_with_none_patterns(): @@ -554,7 +543,7 @@ def test_make_base_create_produces_borg_command(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -601,7 +590,7 @@ def test_make_base_create_command_includes_patterns_file_in_borg_command(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -650,7 +639,7 @@ def test_make_base_create_command_includes_sources_and_config_paths_in_borg_comm config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -697,7 +686,7 @@ def test_make_base_create_command_with_store_config_false_omits_config_files(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -742,7 +731,7 @@ def test_make_base_create_command_includes_exclude_patterns_in_borg_command(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -818,7 +807,7 @@ def test_make_base_create_command_includes_configuration_option_as_command_flag( config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -861,7 +850,7 @@ def test_make_base_create_command_includes_dry_run_in_borg_command(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -903,7 +892,7 @@ def test_make_base_create_command_includes_local_path_in_borg_command(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), local_path='borg1', ) ) @@ -946,7 +935,7 @@ def test_make_base_create_command_includes_remote_path_in_borg_command(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), remote_path='borg1', ) ) @@ -989,7 +978,7 @@ def test_make_base_create_command_includes_log_json_in_borg_command(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=True), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -1031,7 +1020,7 @@ def test_make_base_create_command_includes_list_flags_in_borg_command(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), list_files=True, ) ) @@ -1081,7 +1070,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), stream_processes=flexmock(), ) ) @@ -1127,7 +1116,7 @@ def test_make_base_create_command_with_stream_processes_and_read_special_true_sk config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), stream_processes=flexmock(), ) ) @@ -1140,7 +1129,7 @@ def test_make_base_create_command_with_stream_processes_and_read_special_true_sk def test_make_base_create_command_with_non_matching_source_directories_glob_passes_through(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',)) flexmock(module).should_receive('map_directories_to_devices').and_return({}) flexmock(module).should_receive('expand_directories').and_return(()) @@ -1171,7 +1160,7 @@ def test_make_base_create_command_with_non_matching_source_directories_glob_pass config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -1183,7 +1172,7 @@ def test_make_base_create_command_with_non_matching_source_directories_glob_pass def test_make_base_create_command_expands_glob_in_source_directories(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food')) flexmock(module).should_receive('map_directories_to_devices').and_return({}) flexmock(module).should_receive('expand_directories').and_return(()) @@ -1214,7 +1203,7 @@ def test_make_base_create_command_expands_glob_in_source_directories(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -1226,7 +1215,7 @@ def test_make_base_create_command_expands_glob_in_source_directories(): def test_make_base_create_command_includes_archive_name_format_in_borg_command(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('map_directories_to_devices').and_return({}) flexmock(module).should_receive('expand_directories').and_return(()) @@ -1258,7 +1247,7 @@ def test_make_base_create_command_includes_archive_name_format_in_borg_command() config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -1270,7 +1259,7 @@ def test_make_base_create_command_includes_archive_name_format_in_borg_command() def test_make_base_create_command_includes_default_archive_name_format_in_borg_command(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('map_directories_to_devices').and_return({}) flexmock(module).should_receive('expand_directories').and_return(()) @@ -1301,7 +1290,7 @@ def test_make_base_create_command_includes_default_archive_name_format_in_borg_c config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -1344,7 +1333,7 @@ def test_make_base_create_command_includes_archive_name_format_with_placeholders config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -1387,7 +1376,7 @@ def test_make_base_create_command_includes_repository_and_archive_name_format_wi config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -1430,7 +1419,7 @@ def test_make_base_create_command_includes_extra_borg_options_in_borg_command(): config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) ) @@ -1456,7 +1445,7 @@ def test_make_base_create_command_with_non_existent_directory_and_source_directo config_paths=['/tmp/test.yaml'], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - borgmatic_source_directories=(), + borgmatic_runtime_directories=(), ) @@ -1464,7 +1453,10 @@ def test_create_archive_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1498,7 +1490,10 @@ def test_create_archive_calls_borg_with_environment(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1533,7 +1528,10 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1568,7 +1566,10 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1602,7 +1603,10 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1637,7 +1641,10 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1673,7 +1680,10 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create', '--dry-run'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1709,7 +1719,10 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1746,7 +1759,10 @@ def test_create_archive_with_exit_codes_calls_borg_using_them(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1782,7 +1798,10 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1817,7 +1836,10 @@ def test_create_archive_with_files_calls_borg_with_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( ( ('borg', 'create', '--list', '--filter', 'FOO'), @@ -1857,7 +1879,10 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1893,7 +1918,10 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -1929,7 +1957,10 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER processes = flexmock() flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( ( ('borg', 'create', '--read-special'), @@ -1987,7 +2018,10 @@ def test_create_archive_with_json_calls_borg_with_json_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -2022,7 +2056,10 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) @@ -2058,7 +2095,10 @@ def test_create_archive_calls_borg_with_working_directory(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_runtime_directory' + ).and_return('/var/run/0/borgmatic') + flexmock(module).should_receive('collect_borgmatic_runtime_directories').and_return([]) flexmock(module).should_receive('make_base_create_command').and_return( (('borg', 'create'), REPO_ARCHIVE_WITH_PATHS, flexmock(), flexmock()) ) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index e59bb1eb..d10cf887 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -49,13 +49,20 @@ def test_get_subactions_for_actions_with_subactions_returns_one_entry_per_action ) == {'action': ('foo', 'bar', 'baz'), 'other': ('quux',)} -def test_omit_values_colliding_with_action_names_drops_action_names_that_have__been_parsed_as_values(): +def test_omit_values_colliding_with_action_names_drops_action_names_that_have_been_parsed_as_values(): assert module.omit_values_colliding_with_action_names( ('check', '--only', 'extract', '--some-list', 'borg'), {'check': flexmock(only='extract', some_list=['borg'])}, ) == ('check', '--only', '--some-list') +def test_omit_values_colliding_twice_with_action_names_drops_action_names_that_have_been_parsed_as_values(): + assert module.omit_values_colliding_with_action_names( + ('config', 'bootstrap', '--local-path', '--remote-path', 'borg'), + {'bootstrap': flexmock(local_path='borg', remote_path='borg')}, + ) == ('config', 'bootstrap', '--local-path', '--remote-path') + + def test_parse_and_record_action_arguments_without_action_name_leaves_arguments_untouched(): unparsed_arguments = ('--foo', '--bar') flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return( diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index f0832b45..83c781ca 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1237,7 +1237,7 @@ def test_collect_highlander_action_summary_logs_info_for_success_with_bootstrap( flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap') arguments = { - 'bootstrap': flexmock(repository='repo'), + 'bootstrap': flexmock(repository='repo', local_path='borg7'), 'global': flexmock(dry_run=False), } @@ -1255,7 +1255,7 @@ def test_collect_highlander_action_summary_logs_error_on_bootstrap_failure(): ValueError ) arguments = { - 'bootstrap': flexmock(repository='repo'), + 'bootstrap': flexmock(repository='repo', local_path='borg7'), 'global': flexmock(dry_run=False), } @@ -1272,7 +1272,7 @@ def test_collect_highlander_action_summary_logs_error_on_bootstrap_local_borg_ve flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError) flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').never() arguments = { - 'bootstrap': flexmock(repository='repo'), + 'bootstrap': flexmock(repository='repo', local_path='borg7'), 'global': flexmock(dry_run=False), } diff --git a/tests/unit/config/test_paths.py b/tests/unit/config/test_paths.py index 09086f8c..0b5eddee 100644 --- a/tests/unit/config/test_paths.py +++ b/tests/unit/config/test_paths.py @@ -38,33 +38,27 @@ def test_get_borgmatic_runtime_directory_uses_config_option(): assert ( module.get_borgmatic_runtime_directory( - {'borgmatic_runtime_directory': '/tmp', 'borgmatic_source_directory': '/nope'} + {'user_runtime_directory': '/tmp', 'borgmatic_source_directory': '/nope'} ) - == '/tmp' + == '/tmp/./borgmatic' ) -def test_get_borgmatic_runtime_directory_falls_back_to_borgmatic_source_directory_option(): - flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path) - - assert module.get_borgmatic_runtime_directory({'borgmatic_source_directory': '/tmp'}) == '/tmp' - - def test_get_borgmatic_runtime_directory_falls_back_to_environment_variable(): flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path) flexmock(module.os.environ).should_receive('get').with_args( 'XDG_RUNTIME_DIR', object ).and_return('/tmp') - assert module.get_borgmatic_runtime_directory({}) == '/tmp/borgmatic' + assert module.get_borgmatic_runtime_directory({}) == '/tmp/./borgmatic' def test_get_borgmatic_runtime_directory_defaults_to_hard_coded_path(): flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path) - flexmock(module.os.environ).should_receive('get').and_return('/var/run/0') + flexmock(module.os.environ).should_receive('get').and_return('/run/user/0') flexmock(module.os).should_receive('getuid').and_return(0) - assert module.get_borgmatic_runtime_directory({}) == '/var/run/0/borgmatic' + assert module.get_borgmatic_runtime_directory({}) == '/run/user/0/./borgmatic' def test_get_borgmatic_state_directory_uses_config_option(): @@ -72,18 +66,12 @@ def test_get_borgmatic_state_directory_uses_config_option(): assert ( module.get_borgmatic_state_directory( - {'borgmatic_state_directory': '/tmp', 'borgmatic_source_directory': '/nope'} + {'user_state_directory': '/tmp', 'borgmatic_source_directory': '/nope'} ) - == '/tmp' + == '/tmp/borgmatic' ) -def test_get_borgmatic_state_directory_falls_back_to_borgmatic_source_directory_option(): - flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path) - - assert module.get_borgmatic_state_directory({'borgmatic_source_directory': '/tmp'}) == '/tmp' - - def test_get_borgmatic_state_directory_falls_back_to_environment_variable(): flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path) flexmock(module.os.environ).should_receive('get').with_args( diff --git a/tests/unit/hooks/test_dump.py b/tests/unit/hooks/test_dump.py index bd9f97e7..2db96e6d 100644 --- a/tests/unit/hooks/test_dump.py +++ b/tests/unit/hooks/test_dump.py @@ -8,15 +8,7 @@ def test_make_data_source_dump_path_joins_arguments(): assert module.make_data_source_dump_path('/tmp', 'super_databases') == '/tmp/super_databases' -def test_make_data_source_dump_path_defaults_without_source_directory(): - assert ( - module.make_data_source_dump_path(None, 'super_databases') == '~/.borgmatic/super_databases' - ) - - def test_make_data_source_dump_filename_uses_name_and_hostname(): - flexmock(module.os.path).should_receive('expanduser').and_return('databases') - assert ( module.make_data_source_dump_filename('databases', 'test', 'hostname') == 'databases/hostname/test' @@ -24,14 +16,10 @@ def test_make_data_source_dump_filename_uses_name_and_hostname(): def test_make_data_source_dump_filename_without_hostname_defaults_to_localhost(): - flexmock(module.os.path).should_receive('expanduser').and_return('databases') - assert module.make_data_source_dump_filename('databases', 'test') == 'databases/localhost/test' def test_make_data_source_dump_filename_with_invalid_name_raises(): - flexmock(module.os.path).should_receive('expanduser').and_return('databases') - with pytest.raises(ValueError): module.make_data_source_dump_filename('databases', 'invalid/name') @@ -50,15 +38,13 @@ def test_create_named_pipe_for_dump_does_not_raise(): def test_remove_data_source_dumps_removes_dump_path(): - flexmock(module.os.path).should_receive('expanduser').and_return('databases/localhost') flexmock(module.os.path).should_receive('exists').and_return(True) - flexmock(module.shutil).should_receive('rmtree').with_args('databases/localhost').once() + flexmock(module.shutil).should_receive('rmtree').with_args('databases').once() module.remove_data_source_dumps('databases', 'SuperDB', 'test.yaml', dry_run=False) def test_remove_data_source_dumps_with_dry_run_skips_removal(): - flexmock(module.os.path).should_receive('expanduser').and_return('databases/localhost') flexmock(module.os.path).should_receive('exists').never() flexmock(module.shutil).should_receive('rmtree').never() @@ -66,12 +52,14 @@ def test_remove_data_source_dumps_with_dry_run_skips_removal(): def test_remove_data_source_dumps_without_dump_path_present_skips_removal(): - flexmock(module.os.path).should_receive('expanduser').and_return('databases/localhost') flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.shutil).should_receive('rmtree').never() module.remove_data_source_dumps('databases', 'SuperDB', 'test.yaml', dry_run=False) -def test_convert_glob_patterns_to_borg_patterns_removes_leading_slash(): - assert module.convert_glob_patterns_to_borg_patterns(('/etc/foo/bar',)) == ['sh:etc/foo/bar'] +def test_convert_glob_patterns_to_borg_pattern_makes_multipart_regular_expression(): + assert ( + module.convert_glob_patterns_to_borg_pattern(('/etc/foo/bar', '/bar/*/baz')) + == 're:(?s:etc/foo/bar)|(?s:bar/.*/baz)' + )