From 0e6b2c6773d177bd6d80600dd6946f6e0a52ff54 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 25 Jan 2023 23:31:07 -0800 Subject: [PATCH] Optionally dump "all" PostgreSQL databases to separate files instead of one combined dump file (#438, #560). --- NEWS | 2 +- borgmatic/actions/__init__.py | 0 borgmatic/actions/borg.py | 36 ++ borgmatic/actions/break_lock.py | 21 + borgmatic/actions/check.py | 55 +++ borgmatic/actions/compact.py | 57 +++ borgmatic/actions/create.py | 90 ++++ borgmatic/actions/export_tar.py | 48 ++ borgmatic/actions/extract.py | 67 +++ borgmatic/actions/info.py | 41 ++ borgmatic/actions/list.py | 43 ++ borgmatic/actions/mount.py | 42 ++ borgmatic/actions/prune.py | 53 +++ borgmatic/actions/rcreate.py | 34 ++ borgmatic/actions/restore.py | 345 +++++++++++++++ borgmatic/actions/rinfo.py | 32 ++ borgmatic/actions/rlist.py | 32 ++ borgmatic/actions/transfer.py | 29 ++ borgmatic/borg/list.py | 40 ++ borgmatic/commands/borgmatic.py | 596 +++++-------------------- borgmatic/config/schema.yaml | 13 + borgmatic/execute.py | 3 +- borgmatic/hooks/mysql.py | 122 ++++-- borgmatic/hooks/postgresql.py | 10 +- tests/end-to-end/test_database.py | 10 + tests/unit/actions/__init__.py | 0 tests/unit/actions/test_borg.py | 22 + tests/unit/actions/test_break_lock.py | 19 + tests/unit/actions/test_check.py | 31 ++ tests/unit/actions/test_compact.py | 29 ++ tests/unit/actions/test_create.py | 34 ++ tests/unit/actions/test_export_tar.py | 29 ++ tests/unit/actions/test_extract.py | 33 ++ tests/unit/actions/test_info.py | 24 + tests/unit/actions/test_list.py | 24 + tests/unit/actions/test_mount.py | 26 ++ tests/unit/actions/test_prune.py | 26 ++ tests/unit/actions/test_rcreate.py | 26 ++ tests/unit/actions/test_restore.py | 495 +++++++++++++++++++++ tests/unit/actions/test_rinfo.py | 21 + tests/unit/actions/test_rlist.py | 21 + tests/unit/actions/test_transfer.py | 20 + tests/unit/borg/test_list.py | 13 + tests/unit/commands/test_borgmatic.py | 604 ++++++++++++-------------- tests/unit/hooks/test_mysql.py | 342 +++++++++------ tests/unit/hooks/test_postgresql.py | 27 +- 46 files changed, 2697 insertions(+), 990 deletions(-) create mode 100644 borgmatic/actions/__init__.py create mode 100644 borgmatic/actions/borg.py create mode 100644 borgmatic/actions/break_lock.py create mode 100644 borgmatic/actions/check.py create mode 100644 borgmatic/actions/compact.py create mode 100644 borgmatic/actions/create.py create mode 100644 borgmatic/actions/export_tar.py create mode 100644 borgmatic/actions/extract.py create mode 100644 borgmatic/actions/info.py create mode 100644 borgmatic/actions/list.py create mode 100644 borgmatic/actions/mount.py create mode 100644 borgmatic/actions/prune.py create mode 100644 borgmatic/actions/rcreate.py create mode 100644 borgmatic/actions/restore.py create mode 100644 borgmatic/actions/rinfo.py create mode 100644 borgmatic/actions/rlist.py create mode 100644 borgmatic/actions/transfer.py create mode 100644 tests/unit/actions/__init__.py create mode 100644 tests/unit/actions/test_borg.py create mode 100644 tests/unit/actions/test_break_lock.py create mode 100644 tests/unit/actions/test_check.py create mode 100644 tests/unit/actions/test_compact.py create mode 100644 tests/unit/actions/test_create.py create mode 100644 tests/unit/actions/test_export_tar.py create mode 100644 tests/unit/actions/test_extract.py create mode 100644 tests/unit/actions/test_info.py create mode 100644 tests/unit/actions/test_list.py create mode 100644 tests/unit/actions/test_mount.py create mode 100644 tests/unit/actions/test_prune.py create mode 100644 tests/unit/actions/test_rcreate.py create mode 100644 tests/unit/actions/test_restore.py create mode 100644 tests/unit/actions/test_rinfo.py create mode 100644 tests/unit/actions/test_rlist.py create mode 100644 tests/unit/actions/test_transfer.py diff --git a/NEWS b/NEWS index 8c3acc69..0216afec 100644 --- a/NEWS +++ b/NEWS @@ -3,7 +3,7 @@ dump file, allowing more convenient restores of individual databases. You can enable this by specifying the database dump "format" option when the database is named "all". * #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout. - * #622: Fix traceback when include merging on ARM64. + * #622: Fix traceback when include merging configuration files on ARM64. * #629: Skip warning about excluded special files when no special files have been excluded. 1.7.5 diff --git a/borgmatic/actions/__init__.py b/borgmatic/actions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py new file mode 100644 index 00000000..a50dd286 --- /dev/null +++ b/borgmatic/actions/borg.py @@ -0,0 +1,36 @@ +import logging + +import borgmatic.borg.borg +import borgmatic.borg.rlist +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_borg( + repository, storage, local_borg_version, borg_arguments, local_path, remote_path, +): + ''' + Run the "borg" action for the given repository. + ''' + if borg_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, borg_arguments.repository + ): + logger.info('{}: Running arbitrary Borg command'.format(repository)) + archive_name = borgmatic.borg.rlist.resolve_archive_name( + repository, + borg_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, + ) + borgmatic.borg.borg.run_arbitrary_borg( + repository, + storage, + local_borg_version, + options=borg_arguments.options, + archive=archive_name, + local_path=local_path, + remote_path=remote_path, + ) diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py new file mode 100644 index 00000000..65384d7a --- /dev/null +++ b/borgmatic/actions/break_lock.py @@ -0,0 +1,21 @@ +import logging + +import borgmatic.borg.break_lock +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_break_lock( + repository, storage, local_borg_version, break_lock_arguments, local_path, remote_path, +): + ''' + Run the "break-lock" action for the given repository. + ''' + if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, break_lock_arguments.repository + ): + logger.info(f'{repository}: Breaking repository and cache locks') + borgmatic.borg.break_lock.break_lock( + repository, storage, local_borg_version, local_path=local_path, remote_path=remote_path, + ) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py new file mode 100644 index 00000000..580c70ef --- /dev/null +++ b/borgmatic/actions/check.py @@ -0,0 +1,55 @@ +import logging + +import borgmatic.borg.check +import borgmatic.hooks.command + +logger = logging.getLogger(__name__) + + +def run_check( + config_filename, + repository, + location, + storage, + consistency, + hooks, + hook_context, + local_borg_version, + check_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "check" action for the given repository. + ''' + borgmatic.hooks.command.execute_hook( + hooks.get('before_check'), + hooks.get('umask'), + config_filename, + 'pre-check', + global_arguments.dry_run, + **hook_context, + ) + logger.info('{}: Running consistency checks'.format(repository)) + borgmatic.borg.check.check_archives( + repository, + location, + storage, + consistency, + local_borg_version, + local_path=local_path, + remote_path=remote_path, + progress=check_arguments.progress, + repair=check_arguments.repair, + only_checks=check_arguments.only, + force=check_arguments.force, + ) + borgmatic.hooks.command.execute_hook( + hooks.get('after_check'), + hooks.get('umask'), + config_filename, + 'post-check', + global_arguments.dry_run, + **hook_context, + ) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py new file mode 100644 index 00000000..00585b0b --- /dev/null +++ b/borgmatic/actions/compact.py @@ -0,0 +1,57 @@ +import logging + +import borgmatic.borg.compact +import borgmatic.borg.feature +import borgmatic.hooks.command + +logger = logging.getLogger(__name__) + + +def run_compact( + config_filename, + repository, + storage, + retention, + hooks, + hook_context, + local_borg_version, + compact_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, +): + ''' + Run the "compact" action for the given repository. + ''' + borgmatic.hooks.command.execute_hook( + hooks.get('before_compact'), + hooks.get('umask'), + config_filename, + 'pre-compact', + global_arguments.dry_run, + **hook_context, + ) + if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): + logger.info('{}: Compacting segments{}'.format(repository, dry_run_label)) + borgmatic.borg.compact.compact_segments( + global_arguments.dry_run, + repository, + storage, + local_borg_version, + local_path=local_path, + remote_path=remote_path, + progress=compact_arguments.progress, + cleanup_commits=compact_arguments.cleanup_commits, + threshold=compact_arguments.threshold, + ) + else: # pragma: nocover + logger.info('{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository)) + borgmatic.hooks.command.execute_hook( + hooks.get('after_compact'), + hooks.get('umask'), + config_filename, + 'post-compact', + global_arguments.dry_run, + **hook_context, + ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py new file mode 100644 index 00000000..c882032a --- /dev/null +++ b/borgmatic/actions/create.py @@ -0,0 +1,90 @@ +import json +import logging + +import borgmatic.borg.create +import borgmatic.hooks.command +import borgmatic.hooks.dispatch +import borgmatic.hooks.dump + +logger = logging.getLogger(__name__) + + +def run_create( + config_filename, + repository, + location, + storage, + hooks, + hook_context, + local_borg_version, + create_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, +): + ''' + Run the "create" action for the given repository. + + If create_arguments.json is True, yield the JSON output from creating the archive. + ''' + borgmatic.hooks.command.execute_hook( + hooks.get('before_backup'), + hooks.get('umask'), + config_filename, + 'pre-backup', + global_arguments.dry_run, + **hook_context, + ) + logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) + borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( + 'remove_database_dumps', + hooks, + repository, + borgmatic.hooks.dump.DATABASE_HOOK_NAMES, + location, + global_arguments.dry_run, + ) + active_dumps = borgmatic.hooks.dispatch.call_hooks( + 'dump_databases', + hooks, + repository, + borgmatic.hooks.dump.DATABASE_HOOK_NAMES, + location, + global_arguments.dry_run, + ) + stream_processes = [process for processes in active_dumps.values() for process in processes] + + json_output = borgmatic.borg.create.create_archive( + global_arguments.dry_run, + repository, + location, + storage, + local_borg_version, + local_path=local_path, + remote_path=remote_path, + progress=create_arguments.progress, + stats=create_arguments.stats, + json=create_arguments.json, + list_files=create_arguments.list_files, + stream_processes=stream_processes, + ) + if json_output: # pragma: nocover + yield json.loads(json_output) + + borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( + 'remove_database_dumps', + hooks, + config_filename, + borgmatic.hooks.dump.DATABASE_HOOK_NAMES, + location, + global_arguments.dry_run, + ) + borgmatic.hooks.command.execute_hook( + hooks.get('after_backup'), + hooks.get('umask'), + config_filename, + 'post-backup', + global_arguments.dry_run, + **hook_context, + ) diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py new file mode 100644 index 00000000..ae349208 --- /dev/null +++ b/borgmatic/actions/export_tar.py @@ -0,0 +1,48 @@ +import logging + +import borgmatic.borg.export_tar +import borgmatic.borg.rlist +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_export_tar( + repository, + storage, + local_borg_version, + export_tar_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "export-tar" action for the given repository. + ''' + if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, export_tar_arguments.repository + ): + logger.info( + '{}: Exporting archive {} as tar file'.format(repository, export_tar_arguments.archive) + ) + borgmatic.borg.export_tar.export_tar_archive( + global_arguments.dry_run, + repository, + borgmatic.borg.rlist.resolve_archive_name( + repository, + export_tar_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, + ), + export_tar_arguments.paths, + export_tar_arguments.destination, + storage, + local_borg_version, + local_path=local_path, + remote_path=remote_path, + tar_filter=export_tar_arguments.tar_filter, + list_files=export_tar_arguments.list_files, + strip_components=export_tar_arguments.strip_components, + ) diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py new file mode 100644 index 00000000..a3d89a55 --- /dev/null +++ b/borgmatic/actions/extract.py @@ -0,0 +1,67 @@ +import logging + +import borgmatic.borg.extract +import borgmatic.borg.rlist +import borgmatic.config.validate +import borgmatic.hooks.command + +logger = logging.getLogger(__name__) + + +def run_extract( + config_filename, + repository, + location, + storage, + hooks, + hook_context, + local_borg_version, + extract_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "extract" action for the given repository. + ''' + borgmatic.hooks.command.execute_hook( + hooks.get('before_extract'), + hooks.get('umask'), + config_filename, + 'pre-extract', + global_arguments.dry_run, + **hook_context, + ) + if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, extract_arguments.repository + ): + logger.info('{}: Extracting archive {}'.format(repository, extract_arguments.archive)) + borgmatic.borg.extract.extract_archive( + global_arguments.dry_run, + repository, + borgmatic.borg.rlist.resolve_archive_name( + repository, + extract_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, + ), + extract_arguments.paths, + location, + storage, + local_borg_version, + local_path=local_path, + remote_path=remote_path, + destination_path=extract_arguments.destination, + strip_components=extract_arguments.strip_components, + progress=extract_arguments.progress, + ) + borgmatic.hooks.command.execute_hook( + hooks.get('after_extract'), + hooks.get('umask'), + config_filename, + 'post-extract', + global_arguments.dry_run, + **hook_context, + ) diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py new file mode 100644 index 00000000..ab4fe426 --- /dev/null +++ b/borgmatic/actions/info.py @@ -0,0 +1,41 @@ +import json +import logging + +import borgmatic.borg.info +import borgmatic.borg.rlist +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_info( + repository, storage, local_borg_version, info_arguments, local_path, remote_path, +): + ''' + Run the "info" action for the given repository and archive. + + If info_arguments.json is True, yield the JSON output from the info for the archive. + ''' + if info_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, info_arguments.repository + ): + if not info_arguments.json: # pragma: nocover + logger.answer(f'{repository}: Displaying archive summary information') + info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( + repository, + info_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, + ) + json_output = borgmatic.borg.info.display_archives_info( + repository, + storage, + local_borg_version, + info_arguments=info_arguments, + local_path=local_path, + remote_path=remote_path, + ) + if json_output: # pragma: nocover + yield json.loads(json_output) diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py new file mode 100644 index 00000000..78efdf59 --- /dev/null +++ b/borgmatic/actions/list.py @@ -0,0 +1,43 @@ +import json +import logging + +import borgmatic.borg.list +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_list( + repository, storage, local_borg_version, list_arguments, local_path, remote_path, +): + ''' + Run the "list" action for the given repository and archive. + + If list_arguments.json is True, yield the JSON output from listing the archive. + ''' + if list_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, list_arguments.repository + ): + if not list_arguments.json: # pragma: nocover + if list_arguments.find_paths: + logger.answer(f'{repository}: Searching archives') + elif not list_arguments.archive: + logger.answer(f'{repository}: Listing archives') + list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( + repository, + list_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, + ) + json_output = borgmatic.borg.list.list_archive( + repository, + storage, + local_borg_version, + list_arguments=list_arguments, + local_path=local_path, + remote_path=remote_path, + ) + if json_output: # pragma: nocover + yield json.loads(json_output) diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py new file mode 100644 index 00000000..e2703a57 --- /dev/null +++ b/borgmatic/actions/mount.py @@ -0,0 +1,42 @@ +import logging + +import borgmatic.borg.mount +import borgmatic.borg.rlist +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_mount( + repository, storage, local_borg_version, mount_arguments, local_path, remote_path, +): + ''' + Run the "mount" action for the given repository. + ''' + if mount_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, mount_arguments.repository + ): + if mount_arguments.archive: + logger.info('{}: Mounting archive {}'.format(repository, mount_arguments.archive)) + else: # pragma: nocover + logger.info('{}: Mounting repository'.format(repository)) + + borgmatic.borg.mount.mount_archive( + repository, + borgmatic.borg.rlist.resolve_archive_name( + repository, + mount_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, + ), + mount_arguments.mount_point, + mount_arguments.paths, + mount_arguments.foreground, + mount_arguments.options, + storage, + local_borg_version, + local_path=local_path, + remote_path=remote_path, + ) diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py new file mode 100644 index 00000000..2d214a18 --- /dev/null +++ b/borgmatic/actions/prune.py @@ -0,0 +1,53 @@ +import logging + +import borgmatic.borg.prune +import borgmatic.hooks.command + +logger = logging.getLogger(__name__) + + +def run_prune( + config_filename, + repository, + storage, + retention, + hooks, + hook_context, + local_borg_version, + prune_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, +): + ''' + Run the "prune" action for the given repository. + ''' + borgmatic.hooks.command.execute_hook( + hooks.get('before_prune'), + hooks.get('umask'), + config_filename, + 'pre-prune', + global_arguments.dry_run, + **hook_context, + ) + logger.info('{}: Pruning archives{}'.format(repository, dry_run_label)) + borgmatic.borg.prune.prune_archives( + global_arguments.dry_run, + repository, + storage, + retention, + local_borg_version, + local_path=local_path, + remote_path=remote_path, + stats=prune_arguments.stats, + list_archives=prune_arguments.list_archives, + ) + borgmatic.hooks.command.execute_hook( + hooks.get('after_prune'), + hooks.get('umask'), + config_filename, + 'post-prune', + global_arguments.dry_run, + **hook_context, + ) diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py new file mode 100644 index 00000000..daf4b5b0 --- /dev/null +++ b/borgmatic/actions/rcreate.py @@ -0,0 +1,34 @@ +import logging + +import borgmatic.borg.rcreate + +logger = logging.getLogger(__name__) + + +def run_rcreate( + repository, + storage, + local_borg_version, + rcreate_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "rcreate" action for the given repository. + ''' + logger.info('{}: Creating repository'.format(repository)) + borgmatic.borg.rcreate.create_repository( + global_arguments.dry_run, + repository, + storage, + local_borg_version, + rcreate_arguments.encryption_mode, + rcreate_arguments.source_repository, + rcreate_arguments.copy_crypt_key, + rcreate_arguments.append_only, + rcreate_arguments.storage_quota, + rcreate_arguments.make_parent_dirs, + local_path=local_path, + remote_path=remote_path, + ) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py new file mode 100644 index 00000000..7a058092 --- /dev/null +++ b/borgmatic/actions/restore.py @@ -0,0 +1,345 @@ +import copy +import logging +import os + +import borgmatic.borg.extract +import borgmatic.borg.list +import borgmatic.borg.mount +import borgmatic.borg.rlist +import borgmatic.borg.state +import borgmatic.config.validate +import borgmatic.hooks.dispatch +import borgmatic.hooks.dump + +logger = logging.getLogger(__name__) + + +UNSPECIFIED_HOOK = object() + + +def get_configured_database( + hooks, archive_database_names, hook_name, database_name, configuration_database_name=None +): + ''' + Find the first database with the given hook name and database name in the configured hooks + dict and the given archive database names dict (from hook name to database names contained in + a particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all database + hooks for the named database. If a configuration database name is given, use that instead of the + database name to lookup the database in the given hooks configuration. + + Return the found database as a tuple of (found hook name, database configuration dict). + ''' + if not configuration_database_name: + configuration_database_name = database_name + + if hook_name == UNSPECIFIED_HOOK: + hooks_to_search = hooks + else: + hooks_to_search = {hook_name: hooks[hook_name]} + + return next( + ( + (name, hook_database) + for (name, hook) in hooks_to_search.items() + for hook_database in hook + if hook_database['name'] == configuration_database_name + and database_name in archive_database_names.get(name, []) + ), + (None, None), + ) + + +def get_configured_hook_name_and_database(hooks, database_name): + ''' + Find the hook name and first database dict with the given database name in the configured hooks + dict. This searches across all database hooks. + ''' + + +def restore_single_database( + repository, + location, + storage, + hooks, + local_borg_version, + global_arguments, + local_path, + remote_path, + archive_name, + hook_name, + database, +): # pragma: no cover + ''' + Given (among other things) an archive name, a database hook name, and a configured database + configuration dict, restore that database from the archive. + ''' + logger.info(f'{repository}: Restoring database {database["name"]}') + + dump_pattern = borgmatic.hooks.dispatch.call_hooks( + 'make_database_dump_pattern', + hooks, + repository, + borgmatic.hooks.dump.DATABASE_HOOK_NAMES, + location, + database['name'], + )[hook_name] + + # Kick off a single database extract to stdout. + extract_process = borgmatic.borg.extract.extract_archive( + dry_run=global_arguments.dry_run, + repository=repository, + archive=archive_name, + paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]), + location_config=location, + storage_config=storage, + local_borg_version=local_borg_version, + local_path=local_path, + remote_path=remote_path, + destination_path='/', + # A directory format dump isn't a single file, and therefore can't extract + # to stdout. In this case, the extract_process return value is None. + extract_to_stdout=bool(database.get('format') != 'directory'), + ) + + # Run a single database restore, consuming the extract stdout (if any). + borgmatic.hooks.dispatch.call_hooks( + 'restore_database_dump', + {hook_name: [database]}, + repository, + borgmatic.hooks.dump.DATABASE_HOOK_NAMES, + location, + global_arguments.dry_run, + extract_process, + ) + + +def collect_archive_database_names( + repository, archive, location, storage, local_borg_version, local_path, remote_path, +): + ''' + Given a local or remote repository path, a resolved archive name, a location configuration dict, + a storage configuration dict, the local Borg version, and local and remote Borg paths, query the + archive for the names of databases it contains and return them as a dict from hook name to a + sequence of database names. + ''' + borgmatic_source_directory = os.path.expanduser( + location.get( + 'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY + ) + ).lstrip('/') + parent_dump_path = os.path.expanduser( + borgmatic.hooks.dump.make_database_dump_path(borgmatic_source_directory, '*_databases/*/*') + ) + dump_paths = borgmatic.borg.list.capture_archive_listing( + repository, + archive, + storage, + local_borg_version, + list_path=parent_dump_path, + local_path=local_path, + remote_path=remote_path, + ) + + # Determine the database names corresponding to the dumps found in the archive and + # add them to restore_names. + archive_database_names = {} + + for dump_path in dump_paths: + try: + (hook_name, _, database_name) = dump_path.split( + borgmatic_source_directory + os.path.sep, 1 + )[1].split(os.path.sep)[0:3] + except (ValueError, IndexError): + logger.warning( + f'{repository}: Ignoring invalid database dump path "{dump_path}" in archive {archive}' + ) + else: + if database_name not in archive_database_names.get(hook_name, []): + archive_database_names.setdefault(hook_name, []).extend([database_name]) + + return archive_database_names + + +def find_databases_to_restore(requested_database_names, archive_database_names): + ''' + Given a sequence of requested database names to restore and a dict of hook name to the names of + databases found in an archive, return an expanded sequence of database names to restore, + replacing "all" with actual database names as appropriate. + + Raise ValueError if any of the requested database names cannot be found in the archive. + ''' + # A map from database hook name to the database names to restore for that hook. + restore_names = ( + {UNSPECIFIED_HOOK: requested_database_names} + if requested_database_names + else {UNSPECIFIED_HOOK: ['all']} + ) + + # If "all" is in restore_names, then replace it with the names of dumps found within the + # archive. + if 'all' in restore_names[UNSPECIFIED_HOOK]: + restore_names[UNSPECIFIED_HOOK].remove('all') + + for (hook_name, database_names) in archive_database_names.items(): + restore_names.setdefault(hook_name, []).extend(database_names) + + # If a database is to be restored as part of "all", then remove it from restore names so + # it doesn't get restored twice. + for database_name in database_names: + if database_name in restore_names[UNSPECIFIED_HOOK]: + restore_names[UNSPECIFIED_HOOK].remove(database_name) + + if not restore_names[UNSPECIFIED_HOOK]: + restore_names.pop(UNSPECIFIED_HOOK) + + combined_restore_names = set( + name for database_names in restore_names.values() for name in database_names + ) + combined_archive_database_names = set( + name for database_names in archive_database_names.values() for name in database_names + ) + + missing_names = sorted(set(combined_restore_names) - combined_archive_database_names) + if missing_names: + joined_names = ', '.join(f'"{name}"' for name in missing_names) + raise ValueError( + f"Cannot restore database{'s' if len(missing_names) > 1 else ''} {joined_names} missing from archive" + ) + + return restore_names + + +def ensure_databases_found(restore_names, remaining_restore_names, found_names): + ''' + Given a dict from hook name to database names to restore, a dict from hook name to remaining + database names to restore, and a sequence of found (actually restored) database names, raise + ValueError if requested databases to restore were missing from the archive and/or configuration. + ''' + combined_restore_names = set( + name + for database_names in tuple(restore_names.values()) + + tuple(remaining_restore_names.values()) + for name in database_names + ) + + if not combined_restore_names and not found_names: + raise ValueError('No databases were found to restore') + + missing_names = sorted(set(combined_restore_names) - set(found_names)) + if missing_names: + joined_names = ', '.join(f'"{name}"' for name in missing_names) + raise ValueError( + f"Cannot restore database{'s' if len(missing_names) > 1 else ''} {joined_names} missing from borgmatic's configuration" + ) + + +def run_restore( + repository, + location, + storage, + hooks, + local_borg_version, + restore_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "restore" action for the given repository, but only if the repository matches the + requested repository in restore arguments. + + Raise ValueError if a configured database could not be found to restore. + ''' + if restore_arguments.repository and not borgmatic.config.validate.repositories_match( + repository, restore_arguments.repository + ): + return + + logger.info( + '{}: Restoring databases from archive {}'.format(repository, restore_arguments.archive) + ) + borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( + 'remove_database_dumps', + hooks, + repository, + borgmatic.hooks.dump.DATABASE_HOOK_NAMES, + location, + global_arguments.dry_run, + ) + + archive_name = borgmatic.borg.rlist.resolve_archive_name( + repository, restore_arguments.archive, storage, local_borg_version, local_path, remote_path, + ) + archive_database_names = collect_archive_database_names( + repository, archive_name, location, storage, local_borg_version, local_path, remote_path, + ) + restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names) + found_names = set() + remaining_restore_names = {} + + for hook_name, database_names in restore_names.items(): + for database_name in database_names: + found_hook_name, found_database = get_configured_database( + hooks, archive_database_names, hook_name, database_name + ) + + if not found_database: + remaining_restore_names.setdefault(found_hook_name or hook_name, []).append( + database_name + ) + continue + + found_names.add(database_name) + restore_single_database( + repository, + location, + storage, + hooks, + local_borg_version, + global_arguments, + local_path, + remote_path, + archive_name, + found_hook_name or hook_name, + found_database, + ) + + # For any database that weren't found via exact matches in the hooks configuration, try to + # fallback to "all" entries. + for hook_name, database_names in remaining_restore_names.items(): + for database_name in database_names: + found_hook_name, found_database = get_configured_database( + hooks, archive_database_names, hook_name, database_name, 'all' + ) + + if not found_database: + continue + + found_names.add(database_name) + database = copy.copy(found_database) + database['name'] = database_name + + restore_single_database( + repository, + location, + storage, + hooks, + local_borg_version, + global_arguments, + local_path, + remote_path, + archive_name, + found_hook_name or hook_name, + database, + ) + + borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( + 'remove_database_dumps', + hooks, + repository, + borgmatic.hooks.dump.DATABASE_HOOK_NAMES, + location, + global_arguments.dry_run, + ) + + ensure_databases_found(restore_names, remaining_restore_names, found_names) diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py new file mode 100644 index 00000000..611d1bc2 --- /dev/null +++ b/borgmatic/actions/rinfo.py @@ -0,0 +1,32 @@ +import json +import logging + +import borgmatic.borg.rinfo +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_rinfo( + repository, storage, local_borg_version, rinfo_arguments, local_path, remote_path, +): + ''' + Run the "rinfo" action for the given repository. + + If rinfo_arguments.json is True, yield the JSON output from the info for the repository. + ''' + if rinfo_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, rinfo_arguments.repository + ): + if not rinfo_arguments.json: # pragma: nocover + logger.answer('{}: Displaying repository summary information'.format(repository)) + json_output = borgmatic.borg.rinfo.display_repository_info( + repository, + storage, + local_borg_version, + rinfo_arguments=rinfo_arguments, + local_path=local_path, + remote_path=remote_path, + ) + if json_output: # pragma: nocover + yield json.loads(json_output) diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py new file mode 100644 index 00000000..72d52068 --- /dev/null +++ b/borgmatic/actions/rlist.py @@ -0,0 +1,32 @@ +import json +import logging + +import borgmatic.borg.rlist +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_rlist( + repository, storage, local_borg_version, rlist_arguments, local_path, remote_path, +): + ''' + Run the "rlist" action for the given repository. + + If rlist_arguments.json is True, yield the JSON output from listing the repository. + ''' + if rlist_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, rlist_arguments.repository + ): + if not rlist_arguments.json: # pragma: nocover + logger.answer('{}: Listing repository'.format(repository)) + json_output = borgmatic.borg.rlist.list_repository( + repository, + storage, + local_borg_version, + rlist_arguments=rlist_arguments, + local_path=local_path, + remote_path=remote_path, + ) + if json_output: # pragma: nocover + yield json.loads(json_output) diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py new file mode 100644 index 00000000..628f2735 --- /dev/null +++ b/borgmatic/actions/transfer.py @@ -0,0 +1,29 @@ +import logging + +import borgmatic.borg.transfer + +logger = logging.getLogger(__name__) + + +def run_transfer( + repository, + storage, + local_borg_version, + transfer_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "transfer" action for the given repository. + ''' + logger.info(f'{repository}: Transferring archives to repository') + borgmatic.borg.transfer.transfer_archives( + global_arguments.dry_run, + repository, + storage, + local_borg_version, + transfer_arguments, + local_path=local_path, + remote_path=remote_path, + ) diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 772ab3e0..fedd3650 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -85,6 +85,46 @@ def make_find_paths(find_paths): ) +def capture_archive_listing( + repository, + archive, + storage_config, + local_borg_version, + list_path=None, + local_path='borg', + remote_path=None, +): + ''' + Given a local or remote repository path, an archive name, a storage config dict, the local Borg + version, the archive path in which to list files, and local and remote Borg paths, capture the + output of listing that archive and return it as a list of file paths. + ''' + borg_environment = environment.make_environment(storage_config) + + return tuple( + execute_command_and_capture_output( + make_list_command( + repository, + storage_config, + local_borg_version, + argparse.Namespace( + repository=repository, + archive=archive, + paths=[f'sh:{list_path}'], + find_paths=None, + json=None, + format='{path}{NL}', + ), + local_path, + remote_path, + ), + extra_environment=borg_environment, + ) + .strip('\n') + .split('\n') + ) + + def list_archive( repository, storage_config, diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 292ab262..e08df976 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -1,5 +1,4 @@ import collections -import copy import json import logging import os @@ -11,28 +10,28 @@ from subprocess import CalledProcessError import colorama import pkg_resources +import borgmatic.actions.borg +import borgmatic.actions.break_lock +import borgmatic.actions.check +import borgmatic.actions.compact +import borgmatic.actions.create +import borgmatic.actions.export_tar +import borgmatic.actions.extract +import borgmatic.actions.info +import borgmatic.actions.list +import borgmatic.actions.mount +import borgmatic.actions.prune +import borgmatic.actions.rcreate +import borgmatic.actions.restore +import borgmatic.actions.rinfo +import borgmatic.actions.rlist +import borgmatic.actions.transfer import borgmatic.commands.completion -from borgmatic.borg import borg as borg_borg -from borgmatic.borg import break_lock as borg_break_lock -from borgmatic.borg import check as borg_check -from borgmatic.borg import compact as borg_compact -from borgmatic.borg import create as borg_create -from borgmatic.borg import export_tar as borg_export_tar -from borgmatic.borg import extract as borg_extract -from borgmatic.borg import feature as borg_feature -from borgmatic.borg import info as borg_info -from borgmatic.borg import list as borg_list -from borgmatic.borg import mount as borg_mount -from borgmatic.borg import prune as borg_prune -from borgmatic.borg import rcreate as borg_rcreate -from borgmatic.borg import rinfo as borg_rinfo -from borgmatic.borg import rlist as borg_rlist -from borgmatic.borg import transfer as borg_transfer from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments from borgmatic.config import checks, collect, convert, validate -from borgmatic.hooks import command, dispatch, dump, monitor +from borgmatic.hooks import command, dispatch, monitor from borgmatic.logger import add_custom_log_levels, configure_logging, should_do_markup from borgmatic.signals import configure_signals from borgmatic.verbosity import verbosity_to_log_level @@ -264,509 +263,154 @@ def run_actions( ) if 'rcreate' in arguments: - logger.info('{}: Creating repository'.format(repository)) - borg_rcreate.create_repository( - global_arguments.dry_run, + borgmatic.actions.rcreate.run_rcreate( repository, storage, local_borg_version, - arguments['rcreate'].encryption_mode, - arguments['rcreate'].source_repository, - arguments['rcreate'].copy_crypt_key, - arguments['rcreate'].append_only, - arguments['rcreate'].storage_quota, - arguments['rcreate'].make_parent_dirs, - local_path=local_path, - remote_path=remote_path, + arguments['rcreate'], + global_arguments, + local_path, + remote_path, ) if 'transfer' in arguments: - logger.info(f'{repository}: Transferring archives to repository') - borg_transfer.transfer_archives( - global_arguments.dry_run, + borgmatic.actions.transfer.run_transfer( repository, storage, local_borg_version, - transfer_arguments=arguments['transfer'], - local_path=local_path, - remote_path=remote_path, + arguments['transfer'], + global_arguments, + local_path, + remote_path, ) if 'prune' in arguments: - command.execute_hook( - hooks.get('before_prune'), - hooks.get('umask'), + borgmatic.actions.prune.run_prune( config_filename, - 'pre-prune', - global_arguments.dry_run, - **hook_context, - ) - logger.info('{}: Pruning archives{}'.format(repository, dry_run_label)) - borg_prune.prune_archives( - global_arguments.dry_run, repository, storage, retention, + hooks, + hook_context, local_borg_version, - local_path=local_path, - remote_path=remote_path, - stats=arguments['prune'].stats, - list_archives=arguments['prune'].list_archives, - ) - command.execute_hook( - hooks.get('after_prune'), - hooks.get('umask'), - config_filename, - 'post-prune', - global_arguments.dry_run, - **hook_context, + arguments['prune'], + global_arguments, + dry_run_label, + local_path, + remote_path, ) if 'compact' in arguments: - command.execute_hook( - hooks.get('before_compact'), - hooks.get('umask'), + borgmatic.actions.compact.run_compact( config_filename, - 'pre-compact', - global_arguments.dry_run, - ) - if borg_feature.available(borg_feature.Feature.COMPACT, local_borg_version): - logger.info('{}: Compacting segments{}'.format(repository, dry_run_label)) - borg_compact.compact_segments( - global_arguments.dry_run, - repository, - storage, - local_borg_version, - local_path=local_path, - remote_path=remote_path, - progress=arguments['compact'].progress, - cleanup_commits=arguments['compact'].cleanup_commits, - threshold=arguments['compact'].threshold, - ) - else: # pragma: nocover - logger.info( - '{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository) - ) - command.execute_hook( - hooks.get('after_compact'), - hooks.get('umask'), - config_filename, - 'post-compact', - global_arguments.dry_run, + repository, + storage, + retention, + hooks, + hook_context, + local_borg_version, + arguments['compact'], + global_arguments, + dry_run_label, + local_path, + remote_path, ) if 'create' in arguments: - command.execute_hook( - hooks.get('before_backup'), - hooks.get('umask'), + yield from borgmatic.actions.create.run_create( config_filename, - 'pre-backup', - global_arguments.dry_run, - **hook_context, - ) - logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) - dispatch.call_hooks_even_if_unconfigured( - 'remove_database_dumps', - hooks, - repository, - dump.DATABASE_HOOK_NAMES, - location, - global_arguments.dry_run, - ) - active_dumps = dispatch.call_hooks( - 'dump_databases', - hooks, - repository, - dump.DATABASE_HOOK_NAMES, - location, - global_arguments.dry_run, - ) - stream_processes = [process for processes in active_dumps.values() for process in processes] - - json_output = borg_create.create_archive( - global_arguments.dry_run, repository, location, storage, - local_borg_version, - local_path=local_path, - remote_path=remote_path, - progress=arguments['create'].progress, - stats=arguments['create'].stats, - json=arguments['create'].json, - list_files=arguments['create'].list_files, - stream_processes=stream_processes, - ) - if json_output: # pragma: nocover - yield json.loads(json_output) - - dispatch.call_hooks_even_if_unconfigured( - 'remove_database_dumps', hooks, - config_filename, - dump.DATABASE_HOOK_NAMES, - location, - global_arguments.dry_run, + hook_context, + local_borg_version, + arguments['create'], + global_arguments, + dry_run_label, + local_path, + remote_path, ) - command.execute_hook( - hooks.get('after_backup'), - hooks.get('umask'), - config_filename, - 'post-backup', - global_arguments.dry_run, - **hook_context, - ) - if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency): - command.execute_hook( - hooks.get('before_check'), - hooks.get('umask'), + borgmatic.actions.check.run_check( config_filename, - 'pre-check', - global_arguments.dry_run, - **hook_context, - ) - logger.info('{}: Running consistency checks'.format(repository)) - borg_check.check_archives( repository, location, storage, consistency, + hooks, + hook_context, local_borg_version, - local_path=local_path, - remote_path=remote_path, - progress=arguments['check'].progress, - repair=arguments['check'].repair, - only_checks=arguments['check'].only, - force=arguments['check'].force, - ) - command.execute_hook( - hooks.get('after_check'), - hooks.get('umask'), - config_filename, - 'post-check', - global_arguments.dry_run, - **hook_context, + arguments['check'], + global_arguments, + local_path, + remote_path, ) if 'extract' in arguments: - command.execute_hook( - hooks.get('before_extract'), - hooks.get('umask'), + borgmatic.actions.extract.run_extract( config_filename, - 'pre-extract', - global_arguments.dry_run, - **hook_context, - ) - if arguments['extract'].repository is None or validate.repositories_match( - repository, arguments['extract'].repository - ): - logger.info( - '{}: Extracting archive {}'.format(repository, arguments['extract'].archive) - ) - borg_extract.extract_archive( - global_arguments.dry_run, - repository, - borg_rlist.resolve_archive_name( - repository, - arguments['extract'].archive, - storage, - local_borg_version, - local_path, - remote_path, - ), - arguments['extract'].paths, - location, - storage, - local_borg_version, - local_path=local_path, - remote_path=remote_path, - destination_path=arguments['extract'].destination, - strip_components=arguments['extract'].strip_components, - progress=arguments['extract'].progress, - ) - command.execute_hook( - hooks.get('after_extract'), - hooks.get('umask'), - config_filename, - 'post-extract', - global_arguments.dry_run, - **hook_context, + repository, + location, + storage, + hooks, + hook_context, + local_borg_version, + arguments['extract'], + global_arguments, + local_path, + remote_path, ) if 'export-tar' in arguments: - if arguments['export-tar'].repository is None or validate.repositories_match( - repository, arguments['export-tar'].repository - ): - logger.info( - '{}: Exporting archive {} as tar file'.format( - repository, arguments['export-tar'].archive - ) - ) - borg_export_tar.export_tar_archive( - global_arguments.dry_run, - repository, - borg_rlist.resolve_archive_name( - repository, - arguments['export-tar'].archive, - storage, - local_borg_version, - local_path, - remote_path, - ), - arguments['export-tar'].paths, - arguments['export-tar'].destination, - storage, - local_borg_version, - local_path=local_path, - remote_path=remote_path, - tar_filter=arguments['export-tar'].tar_filter, - list_files=arguments['export-tar'].list_files, - strip_components=arguments['export-tar'].strip_components, - ) + borgmatic.actions.export_tar.run_export_tar( + repository, + storage, + local_borg_version, + arguments['export-tar'], + global_arguments, + local_path, + remote_path, + ) if 'mount' in arguments: - if arguments['mount'].repository is None or validate.repositories_match( - repository, arguments['mount'].repository - ): - if arguments['mount'].archive: - logger.info( - '{}: Mounting archive {}'.format(repository, arguments['mount'].archive) - ) - else: # pragma: nocover - logger.info('{}: Mounting repository'.format(repository)) - - borg_mount.mount_archive( - repository, - borg_rlist.resolve_archive_name( - repository, - arguments['mount'].archive, - storage, - local_borg_version, - local_path, - remote_path, - ), - arguments['mount'].mount_point, - arguments['mount'].paths, - arguments['mount'].foreground, - arguments['mount'].options, - storage, - local_borg_version, - local_path=local_path, - remote_path=remote_path, - ) - if 'restore' in arguments: # pragma: nocover - if arguments['restore'].repository is None or validate.repositories_match( - repository, arguments['restore'].repository - ): - logger.info( - '{}: Restoring databases from archive {}'.format( - repository, arguments['restore'].archive - ) - ) - dispatch.call_hooks_even_if_unconfigured( - 'remove_database_dumps', - hooks, - repository, - dump.DATABASE_HOOK_NAMES, - location, - global_arguments.dry_run, - ) - - restore_names = arguments['restore'].databases or [] - if 'all' in restore_names: - restore_names = [] - - archive_name = borg_rlist.resolve_archive_name( - repository, - arguments['restore'].archive, - storage, - local_borg_version, - local_path, - remote_path, - ) - found_names = set() - - for hook_name, per_hook_restore_databases in hooks.items(): - if hook_name not in dump.DATABASE_HOOK_NAMES: - continue - - for restore_database in per_hook_restore_databases: - database_name = restore_database['name'] - if restore_names and database_name not in restore_names: - continue - - found_names.add(database_name) - dump_pattern = dispatch.call_hooks( - 'make_database_dump_pattern', - hooks, - repository, - dump.DATABASE_HOOK_NAMES, - location, - database_name, - )[hook_name] - - # Kick off a single database extract to stdout. - extract_process = borg_extract.extract_archive( - dry_run=global_arguments.dry_run, - repository=repository, - archive=archive_name, - paths=dump.convert_glob_patterns_to_borg_patterns([dump_pattern]), - location_config=location, - storage_config=storage, - local_borg_version=local_borg_version, - local_path=local_path, - remote_path=remote_path, - destination_path='/', - # A directory format dump isn't a single file, and therefore can't extract - # to stdout. In this case, the extract_process return value is None. - extract_to_stdout=bool(restore_database.get('format') != 'directory'), - ) - - # Run a single database restore, consuming the extract stdout (if any). - dispatch.call_hooks( - 'restore_database_dump', - {hook_name: [restore_database]}, - repository, - dump.DATABASE_HOOK_NAMES, - location, - global_arguments.dry_run, - extract_process, - ) - - dispatch.call_hooks_even_if_unconfigured( - 'remove_database_dumps', - hooks, - repository, - dump.DATABASE_HOOK_NAMES, - location, - global_arguments.dry_run, - ) - - if not restore_names and not found_names: - raise ValueError('No databases were found to restore') - - missing_names = sorted(set(restore_names) - found_names) - if missing_names: - raise ValueError( - 'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format( - ', '.join(missing_names) - ) - ) + borgmatic.actions.mount.run_mount( + repository, storage, local_borg_version, arguments['mount'], local_path, remote_path, + ) + if 'restore' in arguments: + borgmatic.actions.restore.run_restore( + repository, + location, + storage, + hooks, + local_borg_version, + arguments['restore'], + global_arguments, + local_path, + remote_path, + ) if 'rlist' in arguments: - if arguments['rlist'].repository is None or validate.repositories_match( - repository, arguments['rlist'].repository - ): - rlist_arguments = copy.copy(arguments['rlist']) - if not rlist_arguments.json: # pragma: nocover - logger.answer('{}: Listing repository'.format(repository)) - json_output = borg_rlist.list_repository( - repository, - storage, - local_borg_version, - rlist_arguments=rlist_arguments, - local_path=local_path, - remote_path=remote_path, - ) - if json_output: # pragma: nocover - yield json.loads(json_output) + yield from borgmatic.actions.rlist.run_rlist( + repository, storage, local_borg_version, arguments['rlist'], local_path, remote_path, + ) if 'list' in arguments: - if arguments['list'].repository is None or validate.repositories_match( - repository, arguments['list'].repository - ): - list_arguments = copy.copy(arguments['list']) - if not list_arguments.json: # pragma: nocover - if list_arguments.find_paths: - logger.answer('{}: Searching archives'.format(repository)) - elif not list_arguments.archive: - logger.answer('{}: Listing archives'.format(repository)) - list_arguments.archive = borg_rlist.resolve_archive_name( - repository, - list_arguments.archive, - storage, - local_borg_version, - local_path, - remote_path, - ) - json_output = borg_list.list_archive( - repository, - storage, - local_borg_version, - list_arguments=list_arguments, - local_path=local_path, - remote_path=remote_path, - ) - if json_output: # pragma: nocover - yield json.loads(json_output) + yield from borgmatic.actions.list.run_list( + repository, storage, local_borg_version, arguments['list'], local_path, remote_path, + ) if 'rinfo' in arguments: - if arguments['rinfo'].repository is None or validate.repositories_match( - repository, arguments['rinfo'].repository - ): - rinfo_arguments = copy.copy(arguments['rinfo']) - if not rinfo_arguments.json: # pragma: nocover - logger.answer('{}: Displaying repository summary information'.format(repository)) - json_output = borg_rinfo.display_repository_info( - repository, - storage, - local_borg_version, - rinfo_arguments=rinfo_arguments, - local_path=local_path, - remote_path=remote_path, - ) - if json_output: # pragma: nocover - yield json.loads(json_output) + yield from borgmatic.actions.rinfo.run_rinfo( + repository, storage, local_borg_version, arguments['rinfo'], local_path, remote_path, + ) if 'info' in arguments: - if arguments['info'].repository is None or validate.repositories_match( - repository, arguments['info'].repository - ): - info_arguments = copy.copy(arguments['info']) - if not info_arguments.json: # pragma: nocover - logger.answer('{}: Displaying archive summary information'.format(repository)) - info_arguments.archive = borg_rlist.resolve_archive_name( - repository, - info_arguments.archive, - storage, - local_borg_version, - local_path, - remote_path, - ) - json_output = borg_info.display_archives_info( - repository, - storage, - local_borg_version, - info_arguments=info_arguments, - local_path=local_path, - remote_path=remote_path, - ) - if json_output: # pragma: nocover - yield json.loads(json_output) + yield from borgmatic.actions.info.run_info( + repository, storage, local_borg_version, arguments['info'], local_path, remote_path, + ) if 'break-lock' in arguments: - if arguments['break-lock'].repository is None or validate.repositories_match( - repository, arguments['break-lock'].repository - ): - logger.info(f'{repository}: Breaking repository and cache locks') - borg_break_lock.break_lock( - repository, - storage, - local_borg_version, - local_path=local_path, - remote_path=remote_path, - ) + borgmatic.actions.break_lock.run_break_lock( + repository, + storage, + local_borg_version, + arguments['break-lock'], + local_path, + remote_path, + ) if 'borg' in arguments: - if arguments['borg'].repository is None or validate.repositories_match( - repository, arguments['borg'].repository - ): - logger.info('{}: Running arbitrary Borg command'.format(repository)) - archive_name = borg_rlist.resolve_archive_name( - repository, - arguments['borg'].archive, - storage, - local_borg_version, - local_path, - remote_path, - ) - borg_borg.run_arbitrary_borg( - repository, - storage, - local_borg_version, - options=arguments['borg'].options, - archive=archive_name, - local_path=local_path, - remote_path=remote_path, - ) + borgmatic.actions.borg.run_borg( + repository, storage, local_borg_version, arguments['borg'], local_path, remote_path, + ) command.execute_hook( hooks.get('after_actions'), diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 62c16c69..82a9d89b 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -855,6 +855,19 @@ properties: configured to trust the configured username without a password. example: trustsome1 + format: + type: string + enum: ['sql'] + description: | + Database dump output format. Currenly only "sql" + is supported. Defaults to "sql" for a single + database. Or, when database name is "all" and + format is blank, dumps all databases to a single + file. But if a format is specified with an "all" + database name, dumps each database to a separate + file of that format, allowing more convenient + restores of individual databases. + example: directory list_options: type: string description: | diff --git a/borgmatic/execute.py b/borgmatic/execute.py index c3ddfdf4..d4b04bfe 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -49,7 +49,8 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path): ''' Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each process with the requested log level. Additionally, raise a CalledProcessError if a process - exits with an error (or a warning for exit code 1, if that process matches the Borg local path). + exits with an error (or a warning for exit code 1, if that process does not match the Borg local + path). If output log level is None, then instead of logging, capture output for each process and return it as a dict from the process to its output. diff --git a/borgmatic/hooks/mysql.py b/borgmatic/hooks/mysql.py index c7c92c91..41786983 100644 --- a/borgmatic/hooks/mysql.py +++ b/borgmatic/hooks/mysql.py @@ -1,4 +1,6 @@ +import copy import logging +import os from borgmatic.execute import ( execute_command, @@ -28,10 +30,8 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe In the case of "all", query for the names of databases on the configured host and return them, excluding any system databases that will cause problems during restore. ''' - requested_name = database['name'] - - if requested_name != 'all': - return (requested_name,) + if database['name'] != 'all': + return (database['name'],) show_command = ( ('mysql',) @@ -57,6 +57,55 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe ) +def execute_dump_command( + database, log_prefix, dump_path, database_names, extra_environment, dry_run, dry_run_label +): + ''' + Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a + named pipe constructed from the given dump path and database names. Use the given log prefix in + any log entries. + + Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if + this is a dry run, then don't actually dump anything and return None. + ''' + database_name = database['name'] + dump_filename = dump.make_database_dump_filename( + dump_path, database['name'], database.get('hostname') + ) + if os.path.exists(dump_filename): + logger.warning( + f'{log_prefix}: Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}' + ) + return None + + dump_command = ( + ('mysqldump',) + + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + ('--add-drop-database',) + + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--port', str(database['port'])) if 'port' in database else ()) + + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--user', database['username']) if 'username' in database else ()) + + ('--databases',) + + database_names + # Use shell redirection rather than execute_command(output_file=open(...)) to prevent + # the open() call on a named pipe from hanging the main borgmatic process. + + ('>', dump_filename) + ) + + logger.debug( + f'{log_prefix}: Dumping MySQL database "{database_name}" to {dump_filename}{dry_run_label}' + ) + if dry_run: + return None + + dump.create_named_pipe_for_dump(dump_filename) + + return execute_command( + dump_command, shell=True, extra_environment=extra_environment, run_to_completion=False, + ) + + def dump_databases(databases, log_prefix, location_config, dry_run): ''' Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence @@ -73,10 +122,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label)) for database in databases: - requested_name = database['name'] - dump_filename = dump.make_database_dump_filename( - make_dump_path(location_config), requested_name, database.get('hostname') - ) + dump_path = make_dump_path(location_config) extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None dump_database_names = database_names_to_dump( database, extra_environment, log_prefix, dry_run_label @@ -84,41 +130,35 @@ def dump_databases(databases, log_prefix, location_config, dry_run): if not dump_database_names: raise ValueError('Cannot find any MySQL databases to dump.') - dump_command = ( - ('mysqldump',) - + (tuple(database['options'].split(' ')) if 'options' in database else ()) - + ('--add-drop-database',) - + (('--host', database['hostname']) if 'hostname' in database else ()) - + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) - + ('--databases',) - + dump_database_names - # Use shell redirection rather than execute_command(output_file=open(...)) to prevent - # the open() call on a named pipe from hanging the main borgmatic process. - + ('>', dump_filename) - ) - - logger.debug( - '{}: Dumping MySQL database {} to {}{}'.format( - log_prefix, requested_name, dump_filename, dry_run_label + if database['name'] == 'all' and database.get('format'): + for dump_name in dump_database_names: + renamed_database = copy.copy(database) + renamed_database['name'] = dump_name + processes.append( + execute_dump_command( + renamed_database, + log_prefix, + dump_path, + (dump_name,), + extra_environment, + dry_run, + dry_run_label, + ) + ) + else: + processes.append( + execute_dump_command( + database, + log_prefix, + dump_path, + dump_database_names, + extra_environment, + dry_run, + dry_run_label, + ) ) - ) - if dry_run: - continue - dump.create_named_pipe_for_dump(dump_filename) - - processes.append( - execute_command( - dump_command, - shell=True, - extra_environment=extra_environment, - run_to_completion=False, - ) - ) - - return processes + return [process for process in processes if process] def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover diff --git a/borgmatic/hooks/postgresql.py b/borgmatic/hooks/postgresql.py index c184774d..9e3af435 100644 --- a/borgmatic/hooks/postgresql.py +++ b/borgmatic/hooks/postgresql.py @@ -1,5 +1,6 @@ import csv import logging +import os from borgmatic.execute import ( execute_command, @@ -111,6 +112,11 @@ def dump_databases(databases, log_prefix, location_config, dry_run): dump_filename = dump.make_database_dump_filename( dump_path, database_name, database.get('hostname') ) + if os.path.exists(dump_filename): + logger.warning( + f'{log_prefix}: Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}' + ) + continue command = ( (dump_command, '--no-password', '--clean', '--if-exists',) @@ -128,9 +134,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run): ) logger.debug( - '{}: Dumping PostgreSQL database "{}" to {}{}'.format( - log_prefix, database_name, dump_filename, dry_run_label - ) + f'{log_prefix}: Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}' ) if dry_run: continue diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/test_database.py index c6ce0a60..a286af86 100644 --- a/tests/end-to-end/test_database.py +++ b/tests/end-to-end/test_database.py @@ -42,6 +42,11 @@ hooks: hostname: postgresql username: postgres password: test + - name: all + format: custom + hostname: postgresql + username: postgres + password: test mysql_databases: - name: test hostname: mysql @@ -51,6 +56,11 @@ hooks: hostname: mysql username: root password: test + - name: all + format: sql + hostname: mysql + username: root + password: test mongodb_databases: - name: test hostname: mongodb diff --git a/tests/unit/actions/__init__.py b/tests/unit/actions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/actions/test_borg.py b/tests/unit/actions/test_borg.py new file mode 100644 index 00000000..7b22c7b3 --- /dev/null +++ b/tests/unit/actions/test_borg.py @@ -0,0 +1,22 @@ +from flexmock import flexmock + +from borgmatic.actions import borg as module + + +def test_run_borg_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.borg).should_receive('run_arbitrary_borg') + borg_arguments = flexmock(repository=flexmock(), archive=flexmock(), options=flexmock()) + + module.run_borg( + repository='repo', + storage={}, + local_borg_version=None, + borg_arguments=borg_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_break_lock.py b/tests/unit/actions/test_break_lock.py new file mode 100644 index 00000000..c7db00bc --- /dev/null +++ b/tests/unit/actions/test_break_lock.py @@ -0,0 +1,19 @@ +from flexmock import flexmock + +from borgmatic.actions import break_lock as module + + +def test_run_break_lock_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.break_lock).should_receive('break_lock') + break_lock_arguments = flexmock(repository=flexmock()) + + module.run_break_lock( + repository='repo', + storage={}, + local_borg_version=None, + break_lock_arguments=break_lock_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py new file mode 100644 index 00000000..0007ee3f --- /dev/null +++ b/tests/unit/actions/test_check.py @@ -0,0 +1,31 @@ +from flexmock import flexmock + +from borgmatic.actions import check as module + + +def test_run_check_calls_hooks(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.checks).should_receive( + 'repository_enabled_for_checks' + ).and_return(True) + flexmock(module.borgmatic.borg.check).should_receive('check_archives') + flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) + check_arguments = flexmock( + progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock() + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_check( + config_filename='test.yaml', + repository='repo', + location={'repositories': ['repo']}, + storage={}, + consistency={}, + hooks={}, + hook_context={}, + local_borg_version=None, + check_arguments=check_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py new file mode 100644 index 00000000..bc2b9406 --- /dev/null +++ b/tests/unit/actions/test_compact.py @@ -0,0 +1,29 @@ +from flexmock import flexmock + +from borgmatic.actions import compact as module + + +def test_compact_actions_calls_hooks(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.compact).should_receive('compact_segments') + flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) + compact_arguments = flexmock( + progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_compact( + config_filename='test.yaml', + repository='repo', + storage={}, + retention={}, + hooks={}, + hook_context={}, + local_borg_version=None, + compact_arguments=compact_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py new file mode 100644 index 00000000..915f0ae1 --- /dev/null +++ b/tests/unit/actions/test_create.py @@ -0,0 +1,34 @@ +from flexmock import flexmock + +from borgmatic.actions import create as module + + +def test_run_create_executes_and_calls_hooks(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.create).should_receive('create_archive') + flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).and_return({}) + create_arguments = flexmock( + progress=flexmock(), stats=flexmock(), json=flexmock(), list_files=flexmock() + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + list( + module.run_create( + config_filename='test.yaml', + repository='repo', + location={}, + storage={}, + hooks={}, + hook_context={}, + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py new file mode 100644 index 00000000..41b680af --- /dev/null +++ b/tests/unit/actions/test_export_tar.py @@ -0,0 +1,29 @@ +from flexmock import flexmock + +from borgmatic.actions import export_tar as module + + +def test_run_export_tar_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.export_tar).should_receive('export_tar_archive') + export_tar_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + paths=flexmock(), + destination=flexmock(), + tar_filter=flexmock(), + list_files=flexmock(), + strip_components=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_export_tar( + repository='repo', + storage={}, + local_borg_version=None, + export_tar_arguments=export_tar_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py new file mode 100644 index 00000000..4222b8a8 --- /dev/null +++ b/tests/unit/actions/test_extract.py @@ -0,0 +1,33 @@ +from flexmock import flexmock + +from borgmatic.actions import extract as module + + +def test_run_extract_calls_hooks(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive') + flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) + extract_arguments = flexmock( + paths=flexmock(), + progress=flexmock(), + destination=flexmock(), + strip_components=flexmock(), + archive=flexmock(), + repository='repo', + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_extract( + config_filename='test.yaml', + repository='repo', + location={'repositories': ['repo']}, + storage={}, + hooks={}, + hook_context={}, + local_borg_version=None, + extract_arguments=extract_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py new file mode 100644 index 00000000..8cde178d --- /dev/null +++ b/tests/unit/actions/test_info.py @@ -0,0 +1,24 @@ +from flexmock import flexmock + +from borgmatic.actions import info as module + + +def test_run_info_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.info).should_receive('display_archives_info') + info_arguments = flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()) + + list( + module.run_info( + repository='repo', + storage={}, + local_borg_version=None, + info_arguments=info_arguments, + local_path=None, + remote_path=None, + ) + ) diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py new file mode 100644 index 00000000..f4a603de --- /dev/null +++ b/tests/unit/actions/test_list.py @@ -0,0 +1,24 @@ +from flexmock import flexmock + +from borgmatic.actions import list as module + + +def test_run_list_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.list).should_receive('list_archive') + list_arguments = flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()) + + list( + module.run_list( + repository='repo', + storage={}, + local_borg_version=None, + list_arguments=list_arguments, + local_path=None, + remote_path=None, + ) + ) diff --git a/tests/unit/actions/test_mount.py b/tests/unit/actions/test_mount.py new file mode 100644 index 00000000..0624a896 --- /dev/null +++ b/tests/unit/actions/test_mount.py @@ -0,0 +1,26 @@ +from flexmock import flexmock + +from borgmatic.actions import mount as module + + +def test_run_mount_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.mount).should_receive('mount_archive') + mount_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + mount_point=flexmock(), + paths=flexmock(), + foreground=flexmock(), + options=flexmock(), + ) + + module.run_mount( + repository='repo', + storage={}, + local_borg_version=None, + mount_arguments=mount_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py new file mode 100644 index 00000000..d34a9c84 --- /dev/null +++ b/tests/unit/actions/test_prune.py @@ -0,0 +1,26 @@ +from flexmock import flexmock + +from borgmatic.actions import prune as module + + +def test_run_prune_calls_hooks(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.prune).should_receive('prune_archives') + flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) + prune_arguments = flexmock(stats=flexmock(), list_archives=flexmock()) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_prune( + config_filename='test.yaml', + repository='repo', + storage={}, + retention={}, + hooks={}, + hook_context={}, + local_borg_version=None, + prune_arguments=prune_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_rcreate.py b/tests/unit/actions/test_rcreate.py new file mode 100644 index 00000000..8f07a802 --- /dev/null +++ b/tests/unit/actions/test_rcreate.py @@ -0,0 +1,26 @@ +from flexmock import flexmock + +from borgmatic.actions import rcreate as module + + +def test_run_rcreate_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.rcreate).should_receive('create_repository') + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_dirs=flexmock(), + ) + + module.run_rcreate( + repository='repo', + storage={}, + local_borg_version=None, + rcreate_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py new file mode 100644 index 00000000..eaad6bf4 --- /dev/null +++ b/tests/unit/actions/test_restore.py @@ -0,0 +1,495 @@ +import pytest +from flexmock import flexmock + +import borgmatic.actions.restore as module + + +def test_get_configured_database_matches_database_by_name(): + assert module.get_configured_database( + hooks={ + 'other_databases': [{'name': 'other'}], + 'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}], + }, + archive_database_names={'postgresql_databases': ['other', 'foo', 'bar']}, + hook_name='postgresql_databases', + database_name='bar', + ) == ('postgresql_databases', {'name': 'bar'}) + + +def test_get_configured_database_matches_nothing_when_database_name_not_configured(): + assert module.get_configured_database( + hooks={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]}, + archive_database_names={'postgresql_databases': ['foo']}, + hook_name='postgresql_databases', + database_name='quux', + ) == (None, None) + + +def test_get_configured_database_matches_nothing_when_database_name_not_in_archive(): + assert module.get_configured_database( + hooks={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]}, + archive_database_names={'postgresql_databases': ['bar']}, + hook_name='postgresql_databases', + database_name='foo', + ) == (None, None) + + +def test_get_configured_database_matches_database_by_configuration_database_name(): + assert module.get_configured_database( + hooks={'postgresql_databases': [{'name': 'all'}, {'name': 'bar'}]}, + archive_database_names={'postgresql_databases': ['foo']}, + hook_name='postgresql_databases', + database_name='foo', + configuration_database_name='all', + ) == ('postgresql_databases', {'name': 'all'}) + + +def test_get_configured_database_with_unspecified_hook_matches_database_by_name(): + assert module.get_configured_database( + hooks={ + 'other_databases': [{'name': 'other'}], + 'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}], + }, + archive_database_names={'postgresql_databases': ['other', 'foo', 'bar']}, + hook_name=module.UNSPECIFIED_HOOK, + database_name='bar', + ) == ('postgresql_databases', {'name': 'bar'}) + + +def test_collect_archive_database_names_parses_archive_paths(): + flexmock(module.borgmatic.hooks.dump).should_receive('make_database_dump_path').and_return('') + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + [ + '.borgmatic/postgresql_databases/localhost/foo', + '.borgmatic/postgresql_databases/localhost/bar', + '.borgmatic/mysql_databases/localhost/quux', + ] + ) + + archive_database_names = module.collect_archive_database_names( + repository='repo', + archive='archive', + location={'borgmatic_source_directory': '.borgmatic'}, + storage=flexmock(), + local_borg_version=flexmock(), + local_path=flexmock(), + remote_path=flexmock(), + ) + + assert archive_database_names == { + 'postgresql_databases': ['foo', 'bar'], + 'mysql_databases': ['quux'], + } + + +def test_collect_archive_database_names_parses_directory_format_archive_paths(): + flexmock(module.borgmatic.hooks.dump).should_receive('make_database_dump_path').and_return('') + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + [ + '.borgmatic/postgresql_databases/localhost/foo/table1', + '.borgmatic/postgresql_databases/localhost/foo/table2', + ] + ) + + archive_database_names = module.collect_archive_database_names( + repository='repo', + archive='archive', + location={'borgmatic_source_directory': '.borgmatic'}, + storage=flexmock(), + local_borg_version=flexmock(), + local_path=flexmock(), + remote_path=flexmock(), + ) + + assert archive_database_names == { + 'postgresql_databases': ['foo'], + } + + +def test_collect_archive_database_names_skips_bad_archive_paths(): + flexmock(module.borgmatic.hooks.dump).should_receive('make_database_dump_path').and_return('') + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + ['.borgmatic/postgresql_databases/localhost/foo', '.borgmatic/invalid', 'invalid/as/well'] + ) + + archive_database_names = module.collect_archive_database_names( + repository='repo', + archive='archive', + location={'borgmatic_source_directory': '.borgmatic'}, + storage=flexmock(), + local_borg_version=flexmock(), + local_path=flexmock(), + remote_path=flexmock(), + ) + + assert archive_database_names == { + 'postgresql_databases': ['foo'], + } + + +def test_find_databases_to_restore_passes_through_requested_names_found_in_archive(): + restore_names = module.find_databases_to_restore( + requested_database_names=['foo', 'bar'], + archive_database_names={'postresql_databases': ['foo', 'bar', 'baz']}, + ) + + assert restore_names == {module.UNSPECIFIED_HOOK: ['foo', 'bar']} + + +def test_find_databases_to_restore_raises_for_requested_names_missing_from_archive(): + with pytest.raises(ValueError): + module.find_databases_to_restore( + requested_database_names=['foo', 'bar'], + archive_database_names={'postresql_databases': ['foo']}, + ) + + +def test_find_databases_to_restore_without_requested_names_finds_all_archive_databases(): + archive_database_names = {'postresql_databases': ['foo', 'bar']} + + restore_names = module.find_databases_to_restore( + requested_database_names=[], archive_database_names=archive_database_names, + ) + + assert restore_names == archive_database_names + + +def test_find_databases_to_restore_with_all_in_requested_names_finds_all_archive_databases(): + archive_database_names = {'postresql_databases': ['foo', 'bar']} + + restore_names = module.find_databases_to_restore( + requested_database_names=['all'], archive_database_names=archive_database_names, + ) + + assert restore_names == archive_database_names + + +def test_find_databases_to_restore_with_all_in_requested_names_plus_additional_requested_names_omits_duplicates(): + archive_database_names = {'postresql_databases': ['foo', 'bar']} + + restore_names = module.find_databases_to_restore( + requested_database_names=['all', 'foo', 'bar'], + archive_database_names=archive_database_names, + ) + + assert restore_names == archive_database_names + + +def test_find_databases_to_restore_raises_for_all_in_requested_names_and_requested_named_missing_from_archives(): + with pytest.raises(ValueError): + module.find_databases_to_restore( + requested_database_names=['all', 'foo', 'bar'], + archive_database_names={'postresql_databases': ['foo']}, + ) + + +def test_ensure_databases_found_with_all_databases_found_does_not_raise(): + module.ensure_databases_found( + restore_names={'postgresql_databases': ['foo']}, + remaining_restore_names={'postgresql_databases': ['bar']}, + found_names=['foo', 'bar'], + ) + + +def test_ensure_databases_found_with_no_databases_raises(): + with pytest.raises(ValueError): + module.ensure_databases_found( + restore_names={'postgresql_databases': []}, remaining_restore_names={}, found_names=[], + ) + + +def test_ensure_databases_found_with_missing_databases_raises(): + with pytest.raises(ValueError): + module.ensure_databases_found( + restore_names={'postgresql_databases': ['foo']}, + remaining_restore_names={'postgresql_databases': ['bar']}, + found_names=['foo'], + ) + + +def test_run_restore_restores_each_database(): + restore_names = { + 'postgresql_databases': ['foo', 'bar'], + } + + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks_even_if_unconfigured') + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + flexmock() + ) + flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock()) + flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names) + flexmock(module).should_receive('get_configured_database').and_return( + ('postgresql_databases', {'name': 'foo'}) + ).and_return(('postgresql_databases', {'name': 'bar'})) + flexmock(module).should_receive('restore_single_database').with_args( + repository=object, + location=object, + storage=object, + hooks=object, + local_borg_version=object, + global_arguments=object, + local_path=object, + remote_path=object, + archive_name=object, + hook_name='postgresql_databases', + database={'name': 'foo'}, + ).once() + flexmock(module).should_receive('restore_single_database').with_args( + repository=object, + location=object, + storage=object, + hooks=object, + local_borg_version=object, + global_arguments=object, + local_path=object, + remote_path=object, + archive_name=object, + hook_name='postgresql_databases', + database={'name': 'bar'}, + ).once() + flexmock(module).should_receive('ensure_databases_found') + + module.run_restore( + repository='repo', + location=flexmock(), + storage=flexmock(), + hooks=flexmock(), + local_borg_version=flexmock(), + restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + global_arguments=flexmock(dry_run=False), + local_path=flexmock(), + remote_path=flexmock(), + ) + + +def test_run_restore_bails_for_non_matching_repository(): + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( + False + ) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).never() + flexmock(module).should_receive('restore_single_database').never() + + module.run_restore( + repository='repo', + location=flexmock(), + storage=flexmock(), + hooks=flexmock(), + local_borg_version=flexmock(), + restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + global_arguments=flexmock(dry_run=False), + local_path=flexmock(), + remote_path=flexmock(), + ) + + +def test_run_restore_restores_database_configured_with_all_name(): + restore_names = { + 'postgresql_databases': ['foo', 'bar'], + } + + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks_even_if_unconfigured') + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + flexmock() + ) + flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock()) + flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names) + flexmock(module).should_receive('get_configured_database').with_args( + hooks=object, + archive_database_names=object, + hook_name='postgresql_databases', + database_name='foo', + ).and_return(('postgresql_databases', {'name': 'foo'})) + flexmock(module).should_receive('get_configured_database').with_args( + hooks=object, + archive_database_names=object, + hook_name='postgresql_databases', + database_name='bar', + ).and_return((None, None)) + flexmock(module).should_receive('get_configured_database').with_args( + hooks=object, + archive_database_names=object, + hook_name='postgresql_databases', + database_name='bar', + configuration_database_name='all', + ).and_return(('postgresql_databases', {'name': 'bar'})) + flexmock(module).should_receive('restore_single_database').with_args( + repository=object, + location=object, + storage=object, + hooks=object, + local_borg_version=object, + global_arguments=object, + local_path=object, + remote_path=object, + archive_name=object, + hook_name='postgresql_databases', + database={'name': 'foo'}, + ).once() + flexmock(module).should_receive('restore_single_database').with_args( + repository=object, + location=object, + storage=object, + hooks=object, + local_borg_version=object, + global_arguments=object, + local_path=object, + remote_path=object, + archive_name=object, + hook_name='postgresql_databases', + database={'name': 'bar'}, + ).once() + flexmock(module).should_receive('ensure_databases_found') + + module.run_restore( + repository='repo', + location=flexmock(), + storage=flexmock(), + hooks=flexmock(), + local_borg_version=flexmock(), + restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + global_arguments=flexmock(dry_run=False), + local_path=flexmock(), + remote_path=flexmock(), + ) + + +def test_run_restore_skips_missing_database(): + restore_names = { + 'postgresql_databases': ['foo', 'bar'], + } + + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks_even_if_unconfigured') + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + flexmock() + ) + flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock()) + flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names) + flexmock(module).should_receive('get_configured_database').with_args( + hooks=object, + archive_database_names=object, + hook_name='postgresql_databases', + database_name='foo', + ).and_return(('postgresql_databases', {'name': 'foo'})) + flexmock(module).should_receive('get_configured_database').with_args( + hooks=object, + archive_database_names=object, + hook_name='postgresql_databases', + database_name='bar', + ).and_return((None, None)) + flexmock(module).should_receive('get_configured_database').with_args( + hooks=object, + archive_database_names=object, + hook_name='postgresql_databases', + database_name='bar', + configuration_database_name='all', + ).and_return((None, None)) + flexmock(module).should_receive('restore_single_database').with_args( + repository=object, + location=object, + storage=object, + hooks=object, + local_borg_version=object, + global_arguments=object, + local_path=object, + remote_path=object, + archive_name=object, + hook_name='postgresql_databases', + database={'name': 'foo'}, + ).once() + flexmock(module).should_receive('restore_single_database').with_args( + repository=object, + location=object, + storage=object, + hooks=object, + local_borg_version=object, + global_arguments=object, + local_path=object, + remote_path=object, + archive_name=object, + hook_name='postgresql_databases', + database={'name': 'bar'}, + ).never() + flexmock(module).should_receive('ensure_databases_found') + + module.run_restore( + repository='repo', + location=flexmock(), + storage=flexmock(), + hooks=flexmock(), + local_borg_version=flexmock(), + restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + global_arguments=flexmock(dry_run=False), + local_path=flexmock(), + remote_path=flexmock(), + ) + + +def test_run_restore_restores_databases_from_different_hooks(): + restore_names = { + 'postgresql_databases': ['foo'], + 'mysql_databases': ['bar'], + } + + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks_even_if_unconfigured') + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + flexmock() + ) + flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock()) + flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names) + flexmock(module).should_receive('get_configured_database').with_args( + hooks=object, + archive_database_names=object, + hook_name='postgresql_databases', + database_name='foo', + ).and_return(('postgresql_databases', {'name': 'foo'})) + flexmock(module).should_receive('get_configured_database').with_args( + hooks=object, + archive_database_names=object, + hook_name='mysql_databases', + database_name='bar', + ).and_return(('mysql_databases', {'name': 'bar'})) + flexmock(module).should_receive('restore_single_database').with_args( + repository=object, + location=object, + storage=object, + hooks=object, + local_borg_version=object, + global_arguments=object, + local_path=object, + remote_path=object, + archive_name=object, + hook_name='postgresql_databases', + database={'name': 'foo'}, + ).once() + flexmock(module).should_receive('restore_single_database').with_args( + repository=object, + location=object, + storage=object, + hooks=object, + local_borg_version=object, + global_arguments=object, + local_path=object, + remote_path=object, + archive_name=object, + hook_name='mysql_databases', + database={'name': 'bar'}, + ).once() + flexmock(module).should_receive('ensure_databases_found') + + module.run_restore( + repository='repo', + location=flexmock(), + storage=flexmock(), + hooks=flexmock(), + local_borg_version=flexmock(), + restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()), + global_arguments=flexmock(dry_run=False), + local_path=flexmock(), + remote_path=flexmock(), + ) diff --git a/tests/unit/actions/test_rinfo.py b/tests/unit/actions/test_rinfo.py new file mode 100644 index 00000000..d789ef11 --- /dev/null +++ b/tests/unit/actions/test_rinfo.py @@ -0,0 +1,21 @@ +from flexmock import flexmock + +from borgmatic.actions import rinfo as module + + +def test_run_rinfo_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.rinfo).should_receive('display_repository_info') + rinfo_arguments = flexmock(repository=flexmock(), json=flexmock()) + + list( + module.run_rinfo( + repository='repo', + storage={}, + local_borg_version=None, + rinfo_arguments=rinfo_arguments, + local_path=None, + remote_path=None, + ) + ) diff --git a/tests/unit/actions/test_rlist.py b/tests/unit/actions/test_rlist.py new file mode 100644 index 00000000..3da0f905 --- /dev/null +++ b/tests/unit/actions/test_rlist.py @@ -0,0 +1,21 @@ +from flexmock import flexmock + +from borgmatic.actions import rlist as module + + +def test_run_rlist_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.rlist).should_receive('list_repository') + rlist_arguments = flexmock(repository=flexmock(), json=flexmock()) + + list( + module.run_rlist( + repository='repo', + storage={}, + local_borg_version=None, + rlist_arguments=rlist_arguments, + local_path=None, + remote_path=None, + ) + ) diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py new file mode 100644 index 00000000..cc9f1386 --- /dev/null +++ b/tests/unit/actions/test_transfer.py @@ -0,0 +1,20 @@ +from flexmock import flexmock + +from borgmatic.actions import transfer as module + + +def test_run_transfer_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.transfer).should_receive('transfer_archives') + transfer_arguments = flexmock() + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_transfer( + repository='repo', + storage={}, + local_borg_version=None, + transfer_arguments=transfer_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 436ec1b2..cedcec84 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -253,6 +253,19 @@ def test_make_find_paths_adds_globs_to_path_fragments(): assert module.make_find_paths(('foo.txt',)) == ('sh:**/*foo.txt*/**',) +def test_capture_archive_listing_does_not_raise(): + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command_and_capture_output').and_return('') + flexmock(module).should_receive('make_list_command') + + module.capture_archive_listing( + repository='repo', + archive='archive', + storage_config=flexmock(), + local_borg_version=flexmock(), + ) + + def test_list_archive_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 4a9baad6..38b28fd9 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -357,455 +357,387 @@ def test_run_configuration_retries_timeout_multiple_repos(): assert results == error_logs -def test_run_actions_does_not_raise_for_rcreate_action(): +def test_run_actions_runs_rcreate(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.borg_rcreate).should_receive('create_repository') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'rcreate': flexmock( - encryption_mode=flexmock(), - source_repository=flexmock(), - copy_crypt_key=flexmock(), - append_only=flexmock(), - storage_quota=flexmock(), - make_parent_dirs=flexmock(), - ), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.rcreate).should_receive('run_rcreate').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'rcreate': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_does_not_raise_for_transfer_action(): +def test_run_actions_runs_transfer(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.borg_transfer).should_receive('transfer_archives') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'transfer': flexmock(), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.transfer).should_receive('run_transfer').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'transfer': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_calls_hooks_for_prune_action(): +def test_run_actions_runs_prune(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.borg_prune).should_receive('prune_archives') - flexmock(module.command).should_receive('execute_hook').times( - 4 - ) # Before/after extract and before/after actions. - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'prune': flexmock(stats=flexmock(), list_archives=flexmock()), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.prune).should_receive('run_prune').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'prune': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_calls_hooks_for_compact_action(): +def test_run_actions_runs_compact(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.borg_feature).should_receive('available').and_return(True) - flexmock(module.borg_compact).should_receive('compact_segments') - flexmock(module.command).should_receive('execute_hook').times( - 4 - ) # Before/after extract and before/after actions. - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'compact': flexmock(progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock()), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.compact).should_receive('run_compact').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'compact': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_executes_and_calls_hooks_for_create_action(): +def test_run_actions_runs_create(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.borg_create).should_receive('create_archive') - flexmock(module.command).should_receive('execute_hook').times( - 4 - ) # Before/after extract and before/after actions. - flexmock(module.dispatch).should_receive('call_hooks').and_return({}) - flexmock(module.dispatch).should_receive('call_hooks_even_if_unconfigured').and_return({}) - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'create': flexmock( - progress=flexmock(), stats=flexmock(), json=flexmock(), list_files=flexmock() - ), - } + flexmock(module.command).should_receive('execute_hook') + expected = flexmock() + flexmock(borgmatic.actions.create).should_receive('run_create').and_yield(expected).once() - list( + result = tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'create': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) + assert result == (expected,) -def test_run_actions_calls_hooks_for_check_action(): +def test_run_actions_runs_check_when_repository_enabled_for_checks(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None + flexmock(module.command).should_receive('execute_hook') flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True) - flexmock(module.borg_check).should_receive('check_archives') - flexmock(module.command).should_receive('execute_hook').times( - 4 - ) # Before/after extract and before/after actions. - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'check': flexmock( - progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock() - ), - } + flexmock(borgmatic.actions.check).should_receive('run_check').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'check': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_calls_hooks_for_extract_action(): +def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_extract).should_receive('extract_archive') - flexmock(module.command).should_receive('execute_hook').times( - 4 - ) # Before/after extract and before/after actions. - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'extract': flexmock( - paths=flexmock(), - progress=flexmock(), - destination=flexmock(), - strip_components=flexmock(), - archive=flexmock(), - repository='repo', - ), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(False) + flexmock(borgmatic.actions.check).should_receive('run_check').never() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'check': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_does_not_raise_for_export_tar_action(): +def test_run_actions_runs_extract(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_export_tar).should_receive('export_tar_archive') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'export-tar': flexmock( - repository=flexmock(), - archive=flexmock(), - paths=flexmock(), - destination=flexmock(), - tar_filter=flexmock(), - list_files=flexmock(), - strip_components=flexmock(), - ), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.extract).should_receive('run_extract').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'extract': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_does_not_raise_for_mount_action(): +def test_run_actions_runs_export_tar(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_mount).should_receive('mount_archive') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'mount': flexmock( - repository=flexmock(), - archive=flexmock(), - mount_point=flexmock(), - paths=flexmock(), - foreground=flexmock(), - options=flexmock(), - ), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.export_tar).should_receive('run_export_tar').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'export-tar': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_does_not_raise_for_rlist_action(): +def test_run_actions_runs_mount(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_rlist).should_receive('list_repository') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'rlist': flexmock(repository=flexmock(), json=flexmock()), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.mount).should_receive('run_mount').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'mount': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_does_not_raise_for_list_action(): +def test_run_actions_runs_restore(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_rlist).should_receive('resolve_archive_name').and_return(flexmock()) - flexmock(module.borg_list).should_receive('list_archive') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'list': flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.restore).should_receive('run_restore').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'restore': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_does_not_raise_for_rinfo_action(): +def test_run_actions_runs_rlist(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_rinfo).should_receive('display_repository_info') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'rinfo': flexmock(repository=flexmock(), json=flexmock()), - } + flexmock(module.command).should_receive('execute_hook') + expected = flexmock() + flexmock(borgmatic.actions.rlist).should_receive('run_rlist').and_yield(expected).once() - list( + result = tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'rlist': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository_path='repo', + ) + ) + assert result == (expected,) + + +def test_run_actions_runs_list(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + expected = flexmock() + flexmock(borgmatic.actions.list).should_receive('run_list').and_yield(expected).once() + + result = tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False), 'list': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), + hooks={}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository_path='repo', + ) + ) + assert result == (expected,) + + +def test_run_actions_runs_rinfo(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + expected = flexmock() + flexmock(borgmatic.actions.rinfo).should_receive('run_rinfo').and_yield(expected).once() + + result = tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False), 'rinfo': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), + hooks={}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository_path='repo', + ) + ) + assert result == (expected,) + + +def test_run_actions_runs_info(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + expected = flexmock() + flexmock(borgmatic.actions.info).should_receive('run_info').and_yield(expected).once() + + result = tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False), 'info': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), + hooks={}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository_path='repo', + ) + ) + assert result == (expected,) + + +def test_run_actions_runs_break_lock(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.break_lock).should_receive('run_break_lock').once() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False), 'break-lock': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), + hooks={}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) -def test_run_actions_does_not_raise_for_info_action(): +def test_run_actions_runs_borg(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_rlist).should_receive('resolve_archive_name').and_return(flexmock()) - flexmock(module.borg_info).should_receive('display_archives_info') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'info': flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()), - } + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.borg).should_receive('run_borg').once() - list( + tuple( module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, + arguments={'global': flexmock(dry_run=False), 'borg': flexmock()}, + config_filename=flexmock(), + location={'repositories': []}, + storage=flexmock(), + retention=flexmock(), + consistency=flexmock(), hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, - repository_path='repo', - ) - ) - - -def test_run_actions_does_not_raise_for_break_lock_action(): - flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_break_lock).should_receive('break_lock') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'break-lock': flexmock(repository=flexmock()), - } - - list( - module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, - hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, - repository_path='repo', - ) - ) - - -def test_run_actions_does_not_raise_for_borg_action(): - flexmock(module).should_receive('add_custom_log_levels') - flexmock(module.logger).answer = lambda message: None - flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_rlist).should_receive('resolve_archive_name').and_return(flexmock()) - flexmock(module.borg_borg).should_receive('run_arbitrary_borg') - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False), - 'borg': flexmock(repository=flexmock(), archive=flexmock(), options=flexmock()), - } - - list( - module.run_actions( - arguments=arguments, - config_filename='test.yaml', - location={'repositories': ['repo']}, - storage={}, - retention={}, - consistency={}, - hooks={}, - local_path=None, - remote_path=None, - local_borg_version=None, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), repository_path='repo', ) ) diff --git a/tests/unit/hooks/test_mysql.py b/tests/unit/hooks/test_mysql.py index 35516c3c..d9e605b1 100644 --- a/tests/unit/hooks/test_mysql.py +++ b/tests/unit/hooks/test_mysql.py @@ -34,169 +34,87 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): assert names == ('foo', 'bar') -def test_dump_databases_runs_mysqldump_for_each_database(): +def test_dump_databases_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.dump).should_receive('make_database_dump_filename').and_return( - 'databases/localhost/foo' - ).and_return('databases/localhost/bar') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) - flexmock(module.dump).should_receive('create_named_pipe_for_dump') for name, process in zip(('foo', 'bar'), processes): - flexmock(module).should_receive('execute_command').with_args( - ( - 'mysqldump', - '--add-drop-database', - '--databases', - name, - '>', - 'databases/localhost/{}'.format(name), - ), - shell=True, - extra_environment=None, - run_to_completion=False, + flexmock(module).should_receive('execute_dump_command').with_args( + database={'name': name}, + log_prefix=object, + dump_path=object, + database_names=(name,), + extra_environment=object, + dry_run=object, + dry_run_label=object, ).and_return(process).once() assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes -def test_dump_databases_with_dry_run_skips_mysqldump(): - databases = [{'name': 'foo'}, {'name': 'bar'}] +def test_dump_databases_dumps_with_password(): + database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} + process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.dump).should_receive('make_database_dump_filename').and_return( - 'databases/localhost/foo' - ).and_return('databases/localhost/bar') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) - flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() - flexmock(module).should_receive('execute_command').never() - module.dump_databases(databases, 'test.yaml', {}, dry_run=True) - - -def test_dump_databases_runs_mysqldump_with_hostname_and_port(): - databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] - process = flexmock() - flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.dump).should_receive('make_database_dump_filename').and_return( - 'databases/database.example.org/foo' - ) - flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) - flexmock(module.dump).should_receive('create_named_pipe_for_dump') - - flexmock(module).should_receive('execute_command').with_args( - ( - 'mysqldump', - '--add-drop-database', - '--host', - 'database.example.org', - '--port', - '5433', - '--protocol', - 'tcp', - '--databases', - 'foo', - '>', - 'databases/database.example.org/foo', - ), - shell=True, - extra_environment=None, - run_to_completion=False, - ).and_return(process).once() - - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] - - -def test_dump_databases_runs_mysqldump_with_username_and_password(): - databases = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] - process = flexmock() - flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.dump).should_receive('make_database_dump_filename').and_return( - 'databases/localhost/foo' - ) - flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) - flexmock(module.dump).should_receive('create_named_pipe_for_dump') - - flexmock(module).should_receive('execute_command').with_args( - ( - 'mysqldump', - '--add-drop-database', - '--user', - 'root', - '--databases', - 'foo', - '>', - 'databases/localhost/foo', - ), - shell=True, + flexmock(module).should_receive('execute_dump_command').with_args( + database=database, + log_prefix=object, + dump_path=object, + database_names=('foo',), extra_environment={'MYSQL_PWD': 'trustsome1'}, - run_to_completion=False, + dry_run=object, + dry_run_label=object, ).and_return(process).once() - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] + assert module.dump_databases([database], 'test.yaml', {}, dry_run=False) == [process] -def test_dump_databases_runs_mysqldump_with_options(): - databases = [{'name': 'foo', 'options': '--stuff=such'}] - process = flexmock() - flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.dump).should_receive('make_database_dump_filename').and_return( - 'databases/localhost/foo' - ) - flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) - flexmock(module.dump).should_receive('create_named_pipe_for_dump') - - flexmock(module).should_receive('execute_command').with_args( - ( - 'mysqldump', - '--stuff=such', - '--add-drop-database', - '--databases', - 'foo', - '>', - 'databases/localhost/foo', - ), - shell=True, - extra_environment=None, - run_to_completion=False, - ).and_return(process).once() - - assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] - - -def test_dump_databases_runs_mysqldump_for_all_databases(): +def test_dump_databases_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.dump).should_receive('make_database_dump_filename').and_return( - 'databases/localhost/all' - ) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) - flexmock(module.dump).should_receive('create_named_pipe_for_dump') - - flexmock(module).should_receive('execute_command').with_args( - ( - 'mysqldump', - '--add-drop-database', - '--databases', - 'foo', - 'bar', - '>', - 'databases/localhost/all', - ), - shell=True, - extra_environment=None, - run_to_completion=False, + flexmock(module).should_receive('execute_dump_command').with_args( + database={'name': 'all'}, + log_prefix=object, + dump_path=object, + database_names=('foo', 'bar'), + extra_environment=object, + dry_run=object, + dry_run_label=object, ).and_return(process).once() assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process] +def test_dump_databases_dumps_all_databases_separately_when_format_configured(): + databases = [{'name': 'all', 'format': 'sql'}] + processes = [flexmock(), flexmock()] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) + + for name, process in zip(('foo', 'bar'), processes): + flexmock(module).should_receive('execute_dump_command').with_args( + database={'name': name, 'format': 'sql'}, + log_prefix=object, + dump_path=object, + database_names=(name,), + extra_environment=object, + dry_run=object, + dry_run_label=object, + ).and_return(process).once() + + assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes + + def test_database_names_to_dump_runs_mysql_with_list_options(): database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf'} flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -214,6 +132,168 @@ def test_database_names_to_dump_runs_mysql_with_list_options(): assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') +def test_execute_dump_command_runs_mysqldump(): + process = flexmock() + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ('mysqldump', '--add-drop-database', '--databases', 'foo', '>', 'dump',), + shell=True, + extra_environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): + process = flexmock() + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mysqldump', + '--add-drop-database', + '--host', + 'database.example.org', + '--port', + '5433', + '--protocol', + 'tcp', + '--databases', + 'foo', + '>', + 'dump', + ), + shell=True, + extra_environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mysqldump_with_username_and_password(): + process = flexmock() + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ('mysqldump', '--add-drop-database', '--user', 'root', '--databases', 'foo', '>', 'dump',), + shell=True, + extra_environment={'MYSQL_PWD': 'trustsome1'}, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment={'MYSQL_PWD': 'trustsome1'}, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mysqldump_with_options(): + process = flexmock() + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ('mysqldump', '--stuff=such', '--add-drop-database', '--databases', 'foo', '>', 'dump',), + shell=True, + extra_environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'options': '--stuff=such'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() + flexmock(module).should_receive('execute_command').never() + + assert ( + module.execute_dump_command( + database={'name': 'foo'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=True, + dry_run_label='SO DRY', + ) + is None + ) + + +def test_execute_dump_command_with_dry_run_skips_mysqldump(): + flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').never() + + assert ( + module.execute_dump_command( + database={'name': 'foo'}, + log_prefix='log', + dump_path=flexmock(), + database_names=('foo',), + extra_environment=None, + dry_run=True, + dry_run_label='SO DRY', + ) + is None + ) + + def test_dump_databases_errors_for_missing_all_databases(): databases = [{'name': 'all'}] process = flexmock() diff --git a/tests/unit/hooks/test_postgresql.py b/tests/unit/hooks/test_postgresql.py index de5ac04b..5c581828 100644 --- a/tests/unit/hooks/test_postgresql.py +++ b/tests/unit/hooks/test_postgresql.py @@ -56,6 +56,7 @@ def test_dump_databases_runs_pg_dump_for_each_database(): flexmock(module.dump).should_receive('make_database_dump_filename').and_return( 'databases/localhost/foo' ).and_return('databases/localhost/bar') + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') for name, process in zip(('foo', 'bar'), processes): @@ -79,7 +80,7 @@ def test_dump_databases_runs_pg_dump_for_each_database(): assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes -def test_dump_databases_runs_raises_when_no_database_names_to_dump(): +def test_dump_databases_raises_when_no_database_names_to_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -89,6 +90,23 @@ def test_dump_databases_runs_raises_when_no_database_names_to_dump(): module.dump_databases(databases, 'test.yaml', {}, dry_run=False) +def test_dump_databases_with_dupliate_dump_skips_pg_dump(): + databases = [{'name': 'foo'}, {'name': 'bar'}] + flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( + ('bar',) + ) + flexmock(module.dump).should_receive('make_database_dump_filename').and_return( + 'databases/localhost/foo' + ).and_return('databases/localhost/bar') + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() + flexmock(module).should_receive('execute_command').never() + + assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [] + + def test_dump_databases_with_dry_run_skips_pg_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -99,6 +117,7 @@ def test_dump_databases_with_dry_run_skips_pg_dump(): flexmock(module.dump).should_receive('make_database_dump_filename').and_return( 'databases/localhost/foo' ).and_return('databases/localhost/bar') + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -114,6 +133,7 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port(): flexmock(module.dump).should_receive('make_database_dump_filename').and_return( 'databases/database.example.org/foo' ) + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -151,6 +171,7 @@ def test_dump_databases_runs_pg_dump_with_username_and_password(): flexmock(module.dump).should_receive('make_database_dump_filename').and_return( 'databases/localhost/foo' ) + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -207,6 +228,7 @@ def test_dump_databases_runs_pg_dump_with_directory_format(): flexmock(module.dump).should_receive('make_database_dump_filename').and_return( 'databases/localhost/foo' ) + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_parent_directory_for_dump') flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() @@ -239,6 +261,7 @@ def test_dump_databases_runs_pg_dump_with_options(): flexmock(module.dump).should_receive('make_database_dump_filename').and_return( 'databases/localhost/foo' ) + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -271,6 +294,7 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases(): flexmock(module.dump).should_receive('make_database_dump_filename').and_return( 'databases/localhost/all' ) + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -292,6 +316,7 @@ def test_dump_databases_runs_non_default_pg_dump(): flexmock(module.dump).should_receive('make_database_dump_filename').and_return( 'databases/localhost/foo' ) + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args(