From d96f2239c1a50c9376b50fbde570a9b2074da9f0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Mar 2023 23:43:39 -0700 Subject: [PATCH 01/15] Update OpenBSD borgmatic link. --- docs/how-to/set-up-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 52962c342..2ab1e8d57 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -90,7 +90,7 @@ installing borgmatic: * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) - * [OpenBSD](http://ports.su/sysutils/borgmatic) + * [OpenBSD](https://openports.pl/path/sysutils/borgmatic) * [openSUSE](https://software.opensuse.org/package/borgmatic) * [macOS (via Homebrew)](https://formulae.brew.sh/formula/borgmatic) * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/) From 2d08a63e603d53ec2a949405cdccbbf809bcad37 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 28 Mar 2023 22:14:50 +0530 Subject: [PATCH 02/15] fix: make check repositories work with dict and str repositories --- borgmatic/config/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index abcfe3d28..96025b3a2 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -69,7 +69,7 @@ def apply_logical_validation(config_filename, parsed_configuration): location_repositories = parsed_configuration.get('location', {}).get('repositories') check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) for repository in check_repositories: - if repository not in location_repositories: + if not any(repositories_match(repository, config_repository) for config_repository in location_repositories): raise Validation_error( config_filename, ( From ce22d2d30252a23ff8aea0c9afb0491d618aca75 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 28 Mar 2023 22:29:21 +0530 Subject: [PATCH 03/15] reformat --- borgmatic/config/validate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 96025b3a2..fcf29d380 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -69,7 +69,10 @@ def apply_logical_validation(config_filename, parsed_configuration): location_repositories = parsed_configuration.get('location', {}).get('repositories') check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) for repository in check_repositories: - if not any(repositories_match(repository, config_repository) for config_repository in location_repositories): + if not any( + repositories_match(repository, config_repository) + for config_repository in location_repositories + ): raise Validation_error( config_filename, ( From 08e358e27f09b84f08e932973ddf666356c889d4 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Tue, 28 Mar 2023 22:51:35 +0530 Subject: [PATCH 04/15] add and update tests --- tests/unit/config/test_validate.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 327a2b44a..d5606bde1 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -51,17 +51,33 @@ def test_apply_locical_validation_raises_if_unknown_repository_in_check_reposito ) -def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_repositories(): +def test_apply_locical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { - 'location': {'repositories': ['repo.borg', 'other.borg']}, + 'location': {'repositories': [{'path': 'repo.borg'}, {'path': 'other.borg'}]}, 'retention': {'keep_secondly': 1000}, 'consistency': {'check_repositories': ['repo.borg']}, }, ) +def test_apply_locical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): + module.apply_logical_validation( + 'config.yaml', + { + 'location': { + 'repositories': [ + {'path': 'repo.borg', 'label': 'my_repo'}, + {'path': 'other.borg', 'label': 'other_repo'}, + ] + }, + 'retention': {'keep_secondly': 1000}, + 'consistency': {'check_repositories': ['my_repo']}, + }, + ) + + def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present(): module.apply_logical_validation( 'config.yaml', From 59fe01b56d18e92ee49f6b8417c8f867a3f904df Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 11:09:25 -0700 Subject: [PATCH 05/15] Update script comment. --- scripts/run-end-to-end-dev-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run-end-to-end-dev-tests b/scripts/run-end-to-end-dev-tests index 3ee37acc3..575de00d6 100755 --- a/scripts/run-end-to-end-dev-tests +++ b/scripts/run-end-to-end-dev-tests @@ -1,7 +1,7 @@ #!/bin/sh -# This script is for running all tests, including end-to-end tests, on a developer machine. It sets -# up database containers to run tests against, runs the tests, and then tears down the containers. +# This script is for running end-to-end tests on a developer machine. It sets up database containers +# to run tests against, runs the tests, and then tears down the containers. # # Run this script from the root directory of the borgmatic source. # From 3512191f3e8f3c4ac537746dc35f34631e7a19ba Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 11:45:55 -0700 Subject: [PATCH 06/15] Add check_repositories regression fix to NEWS (#662). --- NEWS | 3 +++ setup.py | 2 +- tests/unit/config/test_validate.py | 6 +++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 9bc9ebf6f..7f2353d6f 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.7.11.dev0 + * #662: Fix regression in which "check_repositories" option failed to match repositories. + 1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. * #501: Optionally error if a source directory does not exist via "source_directories_must_exist" diff --git a/setup.py b/setup.py index c36821156..bc4e4b756 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.10' +VERSION = '1.7.11.dev0' setup( diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index d5606bde1..11f3127c8 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -37,7 +37,7 @@ def test_validation_error_string_contains_errors(): assert 'uh oh' in result -def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories(): +def test_apply_logical_validation_raises_if_unknown_repository_in_check_repositories(): flexmock(module).format_json_error = lambda error: error.message with pytest.raises(module.Validation_error): @@ -51,7 +51,7 @@ def test_apply_locical_validation_raises_if_unknown_repository_in_check_reposito ) -def test_apply_locical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): +def test_apply_logical_validation_does_not_raise_if_known_repository_path_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { @@ -62,7 +62,7 @@ def test_apply_locical_validation_does_not_raise_if_known_repository_path_in_che ) -def test_apply_locical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): +def test_apply_logical_validation_does_not_raise_if_known_repository_label_in_check_repositories(): module.apply_logical_validation( 'config.yaml', { From f709125110eb4c58204ed19b3ee591497523d36d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 12:02:07 -0700 Subject: [PATCH 07/15] Error out if run-full-tests is run not inside a test container. --- .drone.yml | 2 ++ scripts/run-full-tests | 9 ++++++++- tests/end-to-end/docker-compose.yaml | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index cf4c358a1..354cff953 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,6 +24,8 @@ clone: steps: - name: build image: alpine:3.13 + environment: + TEST_CONTAINER: true pull: always commands: - scripts/run-full-tests diff --git a/scripts/run-full-tests b/scripts/run-full-tests index cbd824ce2..bf26c2124 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -8,7 +8,14 @@ # For more information, see: # https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/ -set -ex +set -e + +if [ -z "$TEST_CONTAINER" ] ; then + echo "This script is designed to work inside a test container and is not intended to" + echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute" + echo "scripts/run-end-to-end-dev-tests instead." + exit 1 +fi apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \ py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 090cf12c4..884edba0a 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -17,6 +17,8 @@ services: MONGO_INITDB_ROOT_PASSWORD: test tests: image: alpine:3.13 + environment: + TEST_CONTAINER: true volumes: - "../..:/app:ro" tmpfs: From aaf3462d1704a39149708180ab9bf2db9b145244 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 12:03:12 -0700 Subject: [PATCH 08/15] Fix Drone intentation. --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 354cff953..9a7630ace 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,8 +24,8 @@ clone: steps: - name: build image: alpine:3.13 - environment: - TEST_CONTAINER: true + environment: + TEST_CONTAINER: true pull: always commands: - scripts/run-full-tests From 010b82d6d8302616c8fa34973d795afa992f6315 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 12:45:39 -0700 Subject: [PATCH 09/15] Remove unnecessary cd in dev documentation. --- docs/how-to/develop-on-borgmatic.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index d4315457d..796c73673 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -25,7 +25,7 @@ so that you can run borgmatic commands while you're hacking on them to make sure your changes work. ```bash -cd borgmatic/ +cd borgmatic pip3 install --user --editable . ``` @@ -51,7 +51,6 @@ pip3 install --user tox Finally, to actually run tests, run: ```bash -cd borgmatic tox ``` From fc2c181b740b516fab6fb9e9e0907aea5707cc82 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 15:31:37 -0700 Subject: [PATCH 10/15] Add missing Docker Compose depends. --- tests/end-to-end/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end-to-end/docker-compose.yaml b/tests/end-to-end/docker-compose.yaml index 884edba0a..80f12e9a6 100644 --- a/tests/end-to-end/docker-compose.yaml +++ b/tests/end-to-end/docker-compose.yaml @@ -30,3 +30,4 @@ services: depends_on: - postgresql - mysql + - mongodb From b27e625a7772860fb5ab2042c87bcc74b98556d8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Mar 2023 15:44:38 -0700 Subject: [PATCH 11/15] Update schema comment for check_repositories to mention labels (#635). --- borgmatic/config/schema.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3e94b8d2d..469b1f527 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -538,12 +538,12 @@ properties: items: type: string description: | - Paths to a subset of the repositories in the location - section on which to run consistency checks. Handy in case - some of your repositories are very large, and so running - consistency checks on them would take too long. Defaults to - running consistency checks on all repositories configured in - the location section. + Paths or labels for a subset of the repositories in the + location section on which to run consistency checks. Handy + in case some of your repositories are very large, and so + running consistency checks on them would take too long. + Defaults to running consistency checks on all repositories + configured in the location section. example: - user@backupserver:sourcehostname.borg check_last: From 5f595f7ac3fbfc8c19694baaa8b08e613b60a98b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 30 Mar 2023 23:21:20 -0700 Subject: [PATCH 12/15] Fix regression in which the "transfer" action produced a traceback (#663). --- NEWS | 1 + borgmatic/actions/transfer.py | 4 ++-- tests/unit/actions/test_transfer.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 7f2353d6f..8ffbbde32 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.7.11.dev0 * #662: Fix regression in which "check_repositories" option failed to match repositories. + * #663: Fix regression in which the "transfer" action produced a traceback. 1.7.10 * #396: When a database command errors, display and log the error message instead of swallowing it. diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 628f2735a..8089fd4e2 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,10 +17,10 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' - logger.info(f'{repository}: Transferring archives to repository') + logger.info(f'{repository["path"]}: Transferring archives to repository') borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, - repository, + repository['path'], storage, local_borg_version, transfer_arguments, diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py index cc9f1386b..58d8a1603 100644 --- a/tests/unit/actions/test_transfer.py +++ b/tests/unit/actions/test_transfer.py @@ -10,7 +10,7 @@ def test_run_transfer_does_not_raise(): global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_transfer( - repository='repo', + repository={'path': 'repo'}, storage={}, local_borg_version=None, transfer_arguments=transfer_arguments, From 3f78ac4085f689839d9f84d5328cd34b53333e29 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 31 Mar 2023 15:21:08 -0700 Subject: [PATCH 13/15] Automatically use the "archive_name_format" option to filter which archives get used for borgmatic actions that operate on multiple archives (#479). --- NEWS | 5 ++ borgmatic/borg/check.py | 29 +++--- borgmatic/borg/flags.py | 18 ++++ borgmatic/borg/info.py | 6 +- borgmatic/borg/prune.py | 22 ++--- borgmatic/borg/rlist.py | 6 +- borgmatic/borg/transfer.py | 13 ++- borgmatic/config/schema.yaml | 29 +++--- docs/how-to/make-per-application-backups.md | 90 +++++++++++++++---- tests/unit/borg/test_check.py | 99 +++++++++++++-------- tests/unit/borg/test_flags.py | 33 +++++++ tests/unit/borg/test_info.py | 91 ++++++++++++++++++- tests/unit/borg/test_prune.py | 61 +++++++------ tests/unit/borg/test_rlist.py | 69 ++++++++++++++ tests/unit/borg/test_transfer.py | 45 +++++++++- 15 files changed, 488 insertions(+), 128 deletions(-) diff --git a/NEWS b/NEWS index 8ffbbde32..f05188769 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,9 @@ 1.7.11.dev0 + * #479: Automatically use the "archive_name_format" option to filter which archives get used for + borgmatic actions that operate on multiple archives. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming + * #479: The "prefix" options have been deprecated in favor of the new "archive_name_format" + auto-matching behavior (see above). * #662: Fix regression in which "check_repositories" option failed to match repositories. * #663: Fix regression in which the "transfer" action produced a traceback. diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 993a3c0aa..4630782a4 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -12,7 +12,6 @@ DEFAULT_CHECKS = ( {'name': 'repository', 'frequency': '1 month'}, {'name': 'archives', 'frequency': '1 month'}, ) -DEFAULT_PREFIX = '{hostname}-' # noqa: FS003 logger = logging.getLogger(__name__) @@ -146,9 +145,10 @@ def filter_checks_on_frequency( return tuple(filtered_checks) -def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): +def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None): ''' - Given the local Borg version and a parsed sequence of checks, transform the checks into tuple of + Given the local Borg version, a storge configuration dict, a parsed sequence of checks, the + check last value, and a consistency check prefix, transform the checks into tuple of command-line flags. For example, given parsed checks of: @@ -174,10 +174,19 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None): if 'archives' in checks: last_flags = ('--last', str(check_last)) if check_last else () - if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): - match_archives_flags = ('--match-archives', f'sh:{prefix}*') if prefix else () - else: - match_archives_flags = ('--glob-archives', f'{prefix}*') if prefix else () + match_archives_flags = ( + ( + ('--match-archives', f'sh:{prefix}*') + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) + else ('--glob-archives', f'{prefix}*') + ) + if prefix + else ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) + ) else: last_flags = () match_archives_flags = () @@ -291,7 +300,7 @@ def check_archives( extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') if set(checks).intersection({'repository', 'archives', 'data'}): - lock_wait = storage_config.get('lock_wait', None) + lock_wait = storage_config.get('lock_wait') verbosity_flags = () if logger.isEnabledFor(logging.INFO): @@ -299,12 +308,12 @@ def check_archives( if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') - prefix = consistency_config.get('prefix', DEFAULT_PREFIX) + prefix = consistency_config.get('prefix') full_command = ( (local_path, 'check') + (('--repair',) if repair else ()) - + make_check_flags(local_borg_version, checks, check_last, prefix) + + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix) + (('--remote-path', remote_path) if remote_path else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 5dcebf502..1354d6588 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -1,4 +1,5 @@ import itertools +import re from borgmatic.borg import feature @@ -56,3 +57,20 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version): if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version) else (f'{repository_path}::{archive}',) ) + + +def make_match_archives_flags(archive_name_format, local_borg_version): + ''' + Return the match archives flags that would match archives created with the given archive name + format (if any). This is done by replacing certain archive name format placeholders for + ephemeral data (like "{now}") with globs. + ''' + if not archive_name_format: + return () + + match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format) + + if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): + return ('--match-archives', f'sh:{match_archives}') + else: + return ('--glob-archives', f'{match_archives}') diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index 6142104f2..fa59165ca 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -44,7 +44,11 @@ def display_archives_info( else flags.make_flags('glob-archives', f'{info_arguments.prefix}*') ) if info_arguments.prefix - else () + else ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) ) + flags.make_flags_from_arguments( info_arguments, excludes=('repository', 'archive', 'prefix') diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index d21ceee3b..08cb0c176 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -7,10 +7,10 @@ from borgmatic.execute import execute_command logger = logging.getLogger(__name__) -def make_prune_flags(retention_config, local_borg_version): +def make_prune_flags(storage_config, retention_config, local_borg_version): ''' - Given a retention config dict mapping from option name to value, tranform it into an iterable of - command-line name-value flag pairs. + Given a retention config dict mapping from option name to value, tranform it into an sequence of + command-line flags. For example, given a retention config of: @@ -24,7 +24,7 @@ def make_prune_flags(retention_config, local_borg_version): ) ''' config = retention_config.copy() - prefix = config.pop('prefix', '{hostname}-') # noqa: FS003 + prefix = config.pop('prefix', None) if prefix: if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version): @@ -32,10 +32,16 @@ def make_prune_flags(retention_config, local_borg_version): else: config['glob_archives'] = f'{prefix}*' - return ( + flag_pairs = ( ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items() ) + return tuple( + element for pair in flag_pairs for element in pair + ) + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + def prune_archives( dry_run, @@ -60,11 +66,7 @@ def prune_archives( full_command = ( (local_path, 'prune') - + tuple( - element - for pair in make_prune_flags(retention_config, local_borg_version) - for element in pair - ) + + make_prune_flags(storage_config, retention_config, local_borg_version) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 8625363bb..f3935a168 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -94,7 +94,11 @@ def make_rlist_command( else flags.make_flags('glob-archives', f'{rlist_arguments.prefix}*') ) if rlist_arguments.prefix - else () + else ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) ) + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 29e205c77..a350eff77 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -34,9 +34,16 @@ def transfer_archives( 'match-archives', transfer_arguments.match_archives or transfer_arguments.archive ) ) - + flags.make_flags_from_arguments( - transfer_arguments, - excludes=('repository', 'source_repository', 'archive', 'match_archives'), + + ( + flags.make_flags_from_arguments( + transfer_arguments, + excludes=('repository', 'source_repository', 'archive', 'match_archives'), + ) + or ( + flags.make_match_archives_flags( + storage_config.get('archive_name_format'), local_borg_version + ) + ) ) + flags.make_repository_flags(repository_path, local_borg_version) + flags.make_flags('other-repo', transfer_arguments.source_repository) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 469b1f527..d43745254 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -378,11 +378,9 @@ properties: description: | Name of the archive. Borg placeholders can be used. See the output of "borg help placeholders" for details. Defaults to - "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this - option, consider also specifying a prefix in the retention - and consistency sections to avoid accidental - pruning/checking of archives with different archive name - formats. + "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". When running + actions like rlist, info, or check, borgmatic automatically + tries to match only archives created with this name format. example: "{hostname}-documents-{now}" relocated_repo_access_is_ok: type: boolean @@ -477,10 +475,12 @@ properties: prefix: type: string description: | - When pruning, only consider archive names starting with this - prefix. Borg placeholders can be used. See the output of - "borg help placeholders" for details. Defaults to - "{hostname}-". Use an empty value to disable the default. + Deprecated. When pruning, only consider archive names + starting with this prefix. Borg placeholders can be used. + See the output of "borg help placeholders" for details. + If a prefix is not specified, borgmatic defaults to + matching archives based on the archive_name_format (see + above). example: sourcehostname consistency: type: object @@ -556,11 +556,12 @@ properties: prefix: type: string description: | - When performing the "archives" check, only consider archive - names starting with this prefix. Borg placeholders can be - used. See the output of "borg help placeholders" for - details. Defaults to "{hostname}-". Use an empty value to - disable the default. + Deprecated. When performing the "archives" check, only + consider archive names starting with this prefix. Borg + placeholders can be used. See the output of "borg help + placeholders" for details. If a prefix is not specified, + borgmatic defaults to matching archives based on the + archive_name_format (see above). example: sourcehostname output: type: object diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index e5ba037f7..6e5d999ab 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -54,6 +54,72 @@ choice](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot), each entry using borgmatic's `--config` flag instead of relying on `/etc/borgmatic.d`. + +## Archive naming + +If you've got multiple borgmatic configuration files, you might want to create +archives with different naming schemes for each one. This is especially handy +if each configuration file is backing up to the same Borg repository but you +still want to be able to distinguish backup archives for one application from +another. + +borgmatic supports this use case with an `archive_name_format` option. The +idea is that you define a string format containing a number of [Borg +placeholders](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-placeholders), +and borgmatic uses that format to name any new archive it creates. For +instance: + +```yaml +location: + ... + archive_name_format: home-directories-{now} +``` + +This means that when borgmatic creates an archive, its name will start with +the string `home-directories-` and end with a timestamp for its creation time. +If `archive_name_format` is unspecified, the default is +`{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a +timestamp in a particular format. + +New in version 1.7.11 borgmatic +uses the `archive_name_format` option to automatically limit which archives +get used for actions operating on multiple archives. This prevents, for +instance, duplicate archives from showing up in `rlist` or `info` results—even +if the same repository appears in multiple borgmatic configuration files. To +take advantage of this feature, simply use a different `archive_name_format` +in each configuration file. + +Under the hood, borgmatic accomplishes this by substituting globs for certain +ephemeral data placeholders in your `archive_name_format`—and using the result +to filter archives for supported actions. + +For instance, let's say that you have this in your configuration: + +```yaml +location: + ... + archive_name_format: {hostname}-user-data-{now} +``` + +borgmatic considers `{now}` an emphemeral placeholder that will probably +change per archive, while `{hostname}` won't. So it turns the example value +into `{hostname}-user-data-*` and applies it to filter down the set of +archives used for actions like `rlist`, `info`, `prune`, `check`, etc. + +The end result is that when borgmatic runs the actions for a particular +application-specific configuration file, it only operates on the archives +created for that application. Of course, this doesn't apply to actions like +`compact` that operate on an entire repository. + +Prior to 1.7.11 The way to +limit the archives used was a `prefix` option in the `retention` section for +matching against the start of archive names used for a `prune` action and +a separate `prefix` option in the `consistency` section for matching against +the start of archive names used for a `check` action. Both of these options +are deprecated in favor of the auto-matching behavior in newer versions of +borgmatic mentioned above. + + ## Configuration includes Once you have multiple different configuration files, you might want to share @@ -272,7 +338,7 @@ Here's an example usage: ```yaml constants: user: foo - my_prefix: bar- + archive_prefix: bar location: source_directories: @@ -281,20 +347,14 @@ location: ... storage: - archive_name_format: '{my_prefix}{now}' - -retention: - prefix: {my_prefix} - -consistency: - prefix: {my_prefix} + archive_name_format: '{archive_prefix}-{now}' ``` In this example, when borgmatic runs, all instances of `{user}` get replaced -with `foo` and all instances of `{my_prefix}` get replaced with `bar-`. (And -in this particular example, `{now}` doesn't get replaced with anything, but -gets passed directly to Borg.) After substitution, the logical result looks -something like this: +with `foo` and all instances of `{archive-prefix}` get replaced with `bar-`. +(And in this particular example, `{now}` doesn't get replaced with anything, +but gets passed directly to Borg.) After substitution, the logical result +looks something like this: ```yaml location: @@ -305,12 +365,6 @@ location: storage: archive_name_format: 'bar-{now}' - -retention: - prefix: bar- - -consistency: - prefix: bar- ``` An alternate to constants is passing in your values via [environment diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 75755565f..7c233bcfc 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -189,150 +189,170 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_ def test_make_check_flags_with_repository_check_returns_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',)) + flags = module.make_check_flags('1.2.3', {}, ('repository',)) assert flags == ('--repository-only',) def test_make_check_flags_with_archives_check_returns_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',)) + flags = module.make_check_flags('1.2.3', {}, ('archives',)) assert flags == ('--archives-only',) def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',)) + flags = module.make_check_flags('1.2.3', {}, ('data',)) assert flags == ('--archives-only', '--verify-data',) def test_make_check_flags_with_extract_omits_extract_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('extract',)) + flags = module.make_check_flags('1.2.3', {}, ('extract',)) assert flags == () def test_make_check_flags_with_repository_and_data_checks_does_not_return_repository_only(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'data',)) + flags = module.make_check_flags('1.2.3', {}, ('repository', 'data',)) assert flags == ('--verify-data',) -def test_make_check_flags_with_default_checks_and_default_prefix_returns_default_flags(): +def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo',) + + assert flags == ('--match-archives', 'sh:foo*') + + +def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', ('repository', 'archives'), prefix=module.DEFAULT_PREFIX + '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', ) - assert flags == ('--match-archives', f'sh:{module.DEFAULT_PREFIX}*') + assert flags == ('--match-archives', 'sh:foo*') -def test_make_check_flags_with_all_checks_and_default_prefix_returns_default_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - - flags = module.make_check_flags( - '1.2.3', ('repository', 'archives', 'extract'), prefix=module.DEFAULT_PREFIX - ) - - assert flags == ('--match-archives', f'sh:{module.DEFAULT_PREFIX}*') - - -def test_make_check_flags_with_all_checks_and_default_prefix_without_borg_features_returns_glob_archives_flags(): +def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', ('repository', 'archives', 'extract'), prefix=module.DEFAULT_PREFIX + '1.2.3', {}, ('repository', 'archives', 'extract'), prefix='foo', ) - assert flags == ('--glob-archives', f'{module.DEFAULT_PREFIX}*') + assert flags == ('--glob-archives', 'foo*') def test_make_check_flags_with_archives_check_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('archives',), check_last=3) assert flags == ('--archives-only', '--last', '3') def test_make_check_flags_with_data_check_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('data',), check_last=3) assert flags == ('--archives-only', '--last', '3', '--verify-data') def test_make_check_flags_with_repository_check_and_last_omits_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('repository',), check_last=3) assert flags == ('--repository-only',) def test_make_check_flags_with_default_checks_and_last_includes_last_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'archives'), check_last=3) + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) assert flags == ('--last', '3') def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix='foo-') assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('data',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('data',), prefix='foo-') assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data') -def test_make_check_flags_with_archives_check_and_empty_prefix_omits_match_archives_flag(): +def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) - flags = module.make_check_flags('1.2.3', ('archives',), prefix='') + flags = module.make_check_flags( + '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 + ) - assert flags == ('--archives-only',) + assert flags == ('--archives-only', '--match-archives', 'sh:bar-*') def test_make_check_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('archives',), prefix=None) + flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix=None) assert flags == ('--archives-only',) def test_make_check_flags_with_repository_check_and_prefix_omits_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository',), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('repository',), prefix='foo-') assert flags == ('--repository-only',) def test_make_check_flags_with_default_checks_and_prefix_includes_match_archives_flag(): flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', ('repository', 'archives'), prefix='foo-') + flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') assert flags == ('--match-archives', 'sh:foo-*') @@ -427,7 +447,7 @@ def test_check_archives_calls_borg_with_parameters(checks): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', {}, checks, check_last, prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) @@ -581,7 +601,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', {}, checks, check_last, prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1', 'check', 'repo')) @@ -608,7 +628,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', {}, checks, check_last, prefix=None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo')) @@ -628,6 +648,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): checks = ('repository',) check_last = flexmock() + storage_config = {'lock_wait': 5} consistency_config = {'check_last': check_last} flexmock(module).should_receive('parse_checks') flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) @@ -635,7 +656,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, module.DEFAULT_PREFIX + '1.2.3', storage_config, checks, check_last, None, ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo')) @@ -645,7 +666,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): module.check_archives( repository_path='repo', location_config={}, - storage_config={'lock_wait': 5}, + storage_config=storage_config, consistency_config=consistency_config, local_borg_version='1.2.3', ) @@ -662,7 +683,7 @@ def test_check_archives_with_retention_prefix(): '{"repository": {"id": "repo"}}' ) flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', checks, check_last, prefix + '1.2.3', {}, checks, check_last, prefix ).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 1985d8168..cf9eedbba 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -1,3 +1,4 @@ +import pytest from flexmock import flexmock from borgmatic.borg import flags as module @@ -78,3 +79,35 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a assert module.make_repository_archive_flags( repository_path='repo', archive='archive', local_borg_version='1.2.3' ) == ('repo::archive',) + + +@pytest.mark.parametrize( + 'archive_name_format,feature_available,expected_result', + ( + (None, True, ()), + ('', True, ()), + ( + '{hostname}-docs-{now}', # noqa: FS003 + True, + ('--match-archives', 'sh:{hostname}-docs-*'), # noqa: FS003 + ), + ('{utcnow}-docs-{user}', True, ('--match-archives', 'sh:*-docs-{user}')), # noqa: FS003 + ('{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003 + ( + 'stuff-{now:%Y-%m-%dT%H:%M:%S.%f}', # noqa: FS003 + True, + ('--match-archives', 'sh:stuff-*'), + ), + ('{hostname}-docs-{now}', False, ('--glob-archives', '{hostname}-docs-*')), # noqa: FS003 + ('{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003 + ), +) +def test_make_match_archives_flags_makes_flags_with_globs( + archive_name_format, feature_available, expected_result +): + flexmock(module.feature).should_receive('available').and_return(feature_available) + + assert ( + module.make_match_archives_flags(archive_name_format, local_borg_version=flexmock()) + == expected_result + ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index fcc556a40..b25f29f6e 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -12,6 +12,9 @@ def test_display_archives_info_calls_borg_with_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -34,6 +37,9 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -56,6 +62,9 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -78,6 +87,9 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -101,6 +113,9 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -123,6 +138,9 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -147,6 +165,9 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'archive' ).and_return(('--match-archives', 'archive')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -169,6 +190,9 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -195,6 +219,9 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para flexmock(module.flags).should_receive('make_flags').with_args( 'remote-path', 'borg1' ).and_return(('--remote-path', 'borg1')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -221,6 +248,9 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete flexmock(module.flags).should_receive('make_flags').with_args('lock-wait', 5).and_return( ('--lock-wait', '5') ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) storage_config = {'lock_wait': 5} @@ -240,13 +270,16 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete ) -def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parameters(): +def test_display_archives_info_transforms_prefix_into_match_archives_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -265,12 +298,68 @@ def test_display_archives_info_with_prefix_calls_borg_with_match_archives_parame ) +def test_display_archives_info_prefers_prefix_over_archive_name_format(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args( + 'match-archives', 'sh:foo*' + ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:foo*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix='foo'), + ) + + +def test_display_archives_info_transforms_archive_name_format_into_match_archives_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--match-archives', 'sh:bar-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + info_arguments=flexmock(archive=None, json=False, prefix=None), + ) + + @pytest.mark.parametrize('argument_name', ('match_archives', 'sort_by', 'first', 'last')) def test_display_archives_info_passes_through_arguments_to_borg(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '2.3.4' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index f33c9933a..1c8843e2b 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -18,18 +18,17 @@ def insert_execute_command_mock(prune_command, output_log_level): ).once() -BASE_PRUNE_FLAGS = (('--keep-daily', '1'), ('--keep-weekly', '2'), ('--keep-monthly', '3')) +BASE_PRUNE_FLAGS = ('--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') -def test_make_prune_flags_returns_flags_from_config_plus_default_prefix_glob(): +def test_make_prune_flags_returns_flags_from_config(): retention_config = OrderedDict((('keep_daily', 1), ('keep_weekly', 2), ('keep_monthly', 3))) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') - assert tuple(result) == BASE_PRUNE_FLAGS + ( - ('--match-archives', 'sh:{hostname}-*'), # noqa: FS003 - ) + assert result == BASE_PRUNE_FLAGS def test_make_prune_flags_accepts_prefix_with_placeholders(): @@ -37,15 +36,18 @@ def test_make_prune_flags_accepts_prefix_with_placeholders(): (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 ) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') expected = ( - ('--keep-daily', '1'), - ('--match-archives', 'sh:Documents_{hostname}-{now}*'), # noqa: FS003 + '--keep-daily', + '1', + '--match-archives', + 'sh:Documents_{hostname}-{now}*', # noqa: FS003 ) - assert tuple(result) == expected + assert result == expected def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives(): @@ -53,37 +55,38 @@ def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives() (('keep_daily', 1), ('prefix', 'Documents_{hostname}-{now}')) # noqa: FS003 ) flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags({}, retention_config, local_borg_version='1.2.3') expected = ( - ('--keep-daily', '1'), - ('--glob-archives', 'Documents_{hostname}-{now}*'), # noqa: FS003 + '--keep-daily', + '1', + '--glob-archives', + 'Documents_{hostname}-{now}*', # noqa: FS003 ) - assert tuple(result) == expected + assert result == expected -def test_make_prune_flags_treats_empty_prefix_as_no_prefix(): - retention_config = OrderedDict((('keep_daily', 1), ('prefix', ''))) - flexmock(module.feature).should_receive('available').and_return(True) - - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') - - expected = (('--keep-daily', '1'),) - - assert tuple(result) == expected - - -def test_make_prune_flags_treats_none_prefix_as_no_prefix(): +def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): + storage_config = {'archive_name_format': 'bar-{now}'} # noqa: FS003 retention_config = OrderedDict((('keep_daily', 1), ('prefix', None))) flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) - result = module.make_prune_flags(retention_config, local_borg_version='1.2.3') + result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3') - expected = (('--keep-daily', '1'),) + expected = ( + '--keep-daily', + '1', + '--match-archives', + 'sh:bar-*', # noqa: FS003 + ) - assert tuple(result) == expected + assert result == expected PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index a098cafa7..8a10b0756 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -127,6 +127,9 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter def test_make_rlist_command_includes_log_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -143,6 +146,9 @@ def test_make_rlist_command_includes_log_info(): def test_make_rlist_command_includes_json_but_not_info(): insert_logging_mock(logging.INFO) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -159,6 +165,9 @@ def test_make_rlist_command_includes_json_but_not_info(): def test_make_rlist_command_includes_log_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -175,6 +184,9 @@ def test_make_rlist_command_includes_log_debug(): def test_make_rlist_command_includes_json_but_not_debug(): insert_logging_mock(logging.DEBUG) flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -190,6 +202,9 @@ def test_make_rlist_command_includes_json_but_not_debug(): def test_make_rlist_command_includes_json(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -207,6 +222,9 @@ def test_make_rlist_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -222,6 +240,9 @@ def test_make_rlist_command_includes_lock_wait(): def test_make_rlist_command_includes_local_path(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -240,6 +261,9 @@ def test_make_rlist_command_includes_remote_path(): flexmock(module.flags).should_receive('make_flags').and_return( ('--remote-path', 'borg2') ).and_return(()).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -258,6 +282,9 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( ('--match-archives', 'sh:foo*') ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -271,8 +298,47 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') +def test_make_rlist_command_prefers_prefix_over_archive_name_format(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( + ('--match-archives', 'sh:foo*') + ) + flexmock(module.flags).should_receive('make_match_archives_flags').never() + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + ) + + assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') + + +def test_make_rlist_command_transforms_archive_name_format_into_match_archives(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None), + ) + + assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') + + def test_make_rlist_command_includes_short(): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--short',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -301,6 +367,9 @@ def test_make_rlist_command_includes_short(): ) def test_make_rlist_command_includes_additional_flags(argument_name): flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, '1.2.3' + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (f"--{argument_name.replace('_', '-')}", 'value') ) diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index a4814420d..3628a1dc2 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -12,6 +12,7 @@ def test_transfer_archives_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -41,6 +42,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): flexmock(module.flags).should_receive('make_flags').with_args('dry-run', True).and_return( ('--dry-run',) ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -67,6 +69,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -93,6 +96,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -123,6 +127,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'archive' ).and_return(('--match-archives', 'archive')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -137,7 +142,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive='archive', progress=None, match_archives=None, source_repository=None @@ -152,6 +157,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl flexmock(module.flags).should_receive('make_flags').with_args( 'match-archives', 'sh:foo*' ).and_return(('--match-archives', 'sh:foo*')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -166,7 +172,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, repository_path='repo', - storage_config={}, + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None @@ -174,10 +180,40 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl ) +def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archives_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'bar-{now}', '2.3.4' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--match-archives', 'sh:bar-*', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, + borg_local_path='borg', + extra_environment=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + ) + + def test_transfer_archives_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -208,6 +244,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.flags).should_receive('make_flags').with_args( 'remote-path', 'borg2' ).and_return(('--remote-path', 'borg2')) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -238,6 +275,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.flags).should_receive('make_flags').with_args('lock-wait', 5).and_return( ('--lock-wait', '5') ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) storage_config = {'lock_wait': 5} @@ -265,6 +303,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') @@ -293,6 +332,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flag_name = f"--{argument_name.replace('_', ' ')}" flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( (flag_name, 'value') ) @@ -327,6 +367,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla flexmock(module.flags).should_receive('make_flags').with_args('other-repo', 'other').and_return( ('--other-repo', 'other') ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') From f256908b2709b643b4f7b27fd3fda150d3d32bca Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 31 Mar 2023 15:36:59 -0700 Subject: [PATCH 14/15] Document wording tweaks (#479). --- docs/how-to/make-per-application-backups.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 6e5d999ab..fffd2c9df 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -91,7 +91,7 @@ in each configuration file. Under the hood, borgmatic accomplishes this by substituting globs for certain ephemeral data placeholders in your `archive_name_format`—and using the result -to filter archives for supported actions. +to filter archives when running supported actions. For instance, let's say that you have this in your configuration: @@ -101,7 +101,7 @@ location: archive_name_format: {hostname}-user-data-{now} ``` -borgmatic considers `{now}` an emphemeral placeholder that will probably +borgmatic considers `{now}` an emphemeral data placeholder that will probably change per archive, while `{hostname}` won't. So it turns the example value into `{hostname}-user-data-*` and applies it to filter down the set of archives used for actions like `rlist`, `info`, `prune`, `check`, etc. @@ -112,12 +112,11 @@ created for that application. Of course, this doesn't apply to actions like `compact` that operate on an entire repository. Prior to 1.7.11 The way to -limit the archives used was a `prefix` option in the `retention` section for -matching against the start of archive names used for a `prune` action and -a separate `prefix` option in the `consistency` section for matching against -the start of archive names used for a `check` action. Both of these options -are deprecated in favor of the auto-matching behavior in newer versions of -borgmatic mentioned above. +limit the archives used for the `prune` action was a `prefix` option in the +`retention` section for matching against the start of archive names. And the +option for limiting the archives used for the `check` action was a separate +`prefix` in the `consistency` section. Both of these options are deprecated in +favor of the auto-matching behavior in newer versions of borgmatic. ## Configuration includes From fa8bc285c8e82d8fd89327f5877c95957a8b4c76 Mon Sep 17 00:00:00 2001 From: kxxt Date: Sat, 1 Apr 2023 13:45:32 +0800 Subject: [PATCH 15/15] Fix randomly failing test. --- tests/integration/test_execute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 09e9093aa..9c62941b2 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -147,7 +147,7 @@ def test_log_outputs_kills_other_processes_when_one_errors(): ['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) flexmock(module).should_receive('exit_code_indicates_error').with_args( - other_process, None, 'borg' + ['sleep', '2'], None, 'borg' ).and_return(False) flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return( process.stdout