Add "match_archives" option (#588).

This commit is contained in:
Dan Helfman 2023-04-01 23:57:55 -07:00
parent 275e99d0b9
commit 9712d00680
15 changed files with 120 additions and 56 deletions

11
NEWS
View File

@ -1,10 +1,11 @@
1.7.11.dev0
* #479: Automatically use the "archive_name_format" option to filter which archives get used for
borgmatic actions that operate on multiple archives. See the documentation for more information:
* #479, #588: Automatically use the "archive_name_format" option to filter which archives get used
for borgmatic actions that operate on multiple archives. Override this behavior with the new
"match_archives" option in the storage section. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming
* #479: The "prefix" options have been deprecated in favor of the new "archive_name_format"
auto-matching behavior (see above).
* #662: Fix regression in which "check_repositories" option failed to match repositories.
* #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format"
auto-matching behavior and the "match_archives" option.
* #662: Fix regression in which the "check_repositories" option failed to match repositories.
* #663: Fix regression in which the "transfer" action produced a traceback.
* Add spellchecking of source code during test runs.

View File

@ -183,7 +183,9 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None
if prefix
else (
flags.make_match_archives_flags(
storage_config.get('archive_name_format'), local_borg_version
storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
local_borg_version,
)
)
)

View File

@ -59,18 +59,25 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
)
def make_match_archives_flags(archive_name_format, local_borg_version):
def make_match_archives_flags(match_archives, archive_name_format, local_borg_version):
'''
Return the match archives flags that would match archives created with the given archive name
format (if any). This is done by replacing certain archive name format placeholders for
ephemeral data (like "{now}") with globs.
Return match archives flags based on the given match archives value, if any. If it isn't set,
return match archives flags to match archives created with the given archive name format, if
any. This is done by replacing certain archive name format placeholders for ephemeral data (like
"{now}") with globs.
'''
if match_archives:
if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
return ('--match-archives', match_archives)
else:
return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
if not archive_name_format:
return ()
match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format)
derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format)
if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
return ('--match-archives', f'sh:{match_archives}')
return ('--match-archives', f'sh:{derived_match_archives}')
else:
return ('--glob-archives', f'{match_archives}')
return ('--glob-archives', f'{derived_match_archives}')

View File

@ -46,7 +46,9 @@ def display_archives_info(
if info_arguments.prefix
else (
flags.make_match_archives_flags(
storage_config.get('archive_name_format'), local_borg_version
storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
local_borg_version,
)
)
)

View File

@ -39,7 +39,9 @@ def make_prune_flags(storage_config, retention_config, local_borg_version):
return tuple(
element for pair in flag_pairs for element in pair
) + flags.make_match_archives_flags(
storage_config.get('archive_name_format'), local_borg_version
storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
local_borg_version,
)

View File

@ -96,7 +96,9 @@ def make_rlist_command(
if rlist_arguments.prefix
else (
flags.make_match_archives_flags(
storage_config.get('archive_name_format'), local_borg_version
storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
local_borg_version,
)
)
)

View File

@ -41,7 +41,9 @@ def transfer_archives(
)
or (
flags.make_match_archives_flags(
storage_config.get('archive_name_format'), local_borg_version
storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
local_borg_version,
)
)
)

View File

@ -382,6 +382,17 @@ properties:
actions like rlist, info, or check, borgmatic automatically
tries to match only archives created with this name format.
example: "{hostname}-documents-{now}"
match_archives:
type: string
description: |
A Borg pattern for filtering down the archives used by
borgmatic actions that operate on multiple archives. For
Borg 1.x, use a shell pattern here and see the output of
"borg help placeholders" for details. For Borg 2.x, see the
output of "borg help match-archives". If match_archives is
not specified, borgmatic defaults to deriving the
match_archives value from archive_name_format.
example: "sh:{hostname}-*"
relocated_repo_access_is_ok:
type: boolean
description: |

View File

@ -111,12 +111,30 @@ application-specific configuration file, it only operates on the archives
created for that application. Of course, this doesn't apply to actions like
`compact` that operate on an entire repository.
If this behavior isn't quite smart enough for your needs, you can use the
`match_archives` option to override the pattern that borgmatic uses for
filtering archives. For example:
```yaml
location:
...
archive_name_format: {hostname}-user-data-{now}
match_archives: sh:myhost-user-data-*
```
For Borg 1.x, use a shell pattern for the `match_archives` value and see the
[Borg patterns
documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns)
for more information. For Borg 2.x, see the [match archives
documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/help.html#borg-help-match-archives).
<span class="minilink minilink-addedin">Prior to 1.7.11</span> The way to
limit the archives used for the `prune` action was a `prefix` option in the
`retention` section for matching against the start of archive names. And the
option for limiting the archives used for the `check` action was a separate
`prefix` in the `consistency` section. Both of these options are deprecated in
favor of the auto-matching behavior in newer versions of borgmatic.
favor of the auto-matching behavior (or `match_archives`) in newer versions of
borgmatic.
## Configuration includes

View File

@ -320,7 +320,7 @@ def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_fla
def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
'bar-{now}', '1.2.3' # noqa: FS003
None, 'bar-{now}', '1.2.3' # noqa: FS003
).and_return(('--match-archives', 'sh:bar-*'))
flags = module.make_check_flags(

View File

@ -82,32 +82,49 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a
@pytest.mark.parametrize(
'archive_name_format,feature_available,expected_result',
'match_archives, archive_name_format,feature_available,expected_result',
(
(None, True, ()),
('', True, ()),
(None, None, True, ()),
(None, '', True, ()),
('re:foo-.*', '{hostname}-{now}', True, ('--match-archives', 're:foo-.*'),), # noqa: FS003
('sh:foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003
('foo-*', '{hostname}-{now}', False, ('--glob-archives', 'foo-*'),), # noqa: FS003
(
None,
'{hostname}-docs-{now}', # noqa: FS003
True,
('--match-archives', 'sh:{hostname}-docs-*'), # noqa: FS003
),
('{utcnow}-docs-{user}', True, ('--match-archives', 'sh:*-docs-{user}')), # noqa: FS003
('{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003
(
None,
'{utcnow}-docs-{user}', # noqa: FS003
True,
('--match-archives', 'sh:*-docs-{user}'), # noqa: FS003
),
(None, '{fqdn}-{pid}', True, ('--match-archives', 'sh:{fqdn}-*')), # noqa: FS003
(
None,
'stuff-{now:%Y-%m-%dT%H:%M:%S.%f}', # noqa: FS003
True,
('--match-archives', 'sh:stuff-*'),
),
('{hostname}-docs-{now}', False, ('--glob-archives', '{hostname}-docs-*')), # noqa: FS003
('{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003
(
None,
'{hostname}-docs-{now}', # noqa: FS003
False,
('--glob-archives', '{hostname}-docs-*'), # noqa: FS003
),
(None, '{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')), # noqa: FS003
),
)
def test_make_match_archives_flags_makes_flags_with_globs(
archive_name_format, feature_available, expected_result
match_archives, archive_name_format, feature_available, expected_result
):
flexmock(module.feature).should_receive('available').and_return(feature_available)
assert (
module.make_match_archives_flags(archive_name_format, local_borg_version=flexmock())
module.make_match_archives_flags(
match_archives, archive_name_format, local_borg_version=flexmock()
)
== expected_result
)

View File

@ -13,7 +13,7 @@ def test_display_archives_info_calls_borg_with_parameters():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -38,7 +38,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -63,7 +63,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',))
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -88,7 +88,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -114,7 +114,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',))
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -139,7 +139,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',))
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -166,7 +166,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param
'match-archives', 'archive'
).and_return(('--match-archives', 'archive'))
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -191,7 +191,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -220,7 +220,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para
'remote-path', 'borg1'
).and_return(('--remote-path', 'borg1'))
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -249,7 +249,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete
('--lock-wait', '5')
)
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -278,7 +278,7 @@ def test_display_archives_info_transforms_prefix_into_match_archives_parameters(
'match-archives', 'sh:foo*'
).and_return(('--match-archives', 'sh:foo*'))
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -306,7 +306,7 @@ def test_display_archives_info_prefers_prefix_over_archive_name_format():
'match-archives', 'sh:foo*'
).and_return(('--match-archives', 'sh:foo*'))
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -331,7 +331,7 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
'bar-{now}', '2.3.4' # noqa: FS003
None, 'bar-{now}', '2.3.4' # noqa: FS003
).and_return(('--match-archives', 'sh:bar-*'))
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
@ -358,7 +358,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
flag_name = f"--{argument_name.replace('_', ' ')}"
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '2.3.4'
None, None, '2.3.4'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(
(flag_name, 'value')

View File

@ -74,7 +74,7 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead():
retention_config = OrderedDict((('keep_daily', 1), ('prefix', None)))
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
'bar-{now}', '1.2.3' # noqa: FS003
None, 'bar-{now}', '1.2.3' # noqa: FS003
).and_return(('--match-archives', 'sh:bar-*'))
result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3')

View File

@ -128,7 +128,7 @@ def test_make_rlist_command_includes_log_info():
insert_logging_mock(logging.INFO)
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -147,7 +147,7 @@ def test_make_rlist_command_includes_json_but_not_info():
insert_logging_mock(logging.INFO)
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',))
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -166,7 +166,7 @@ def test_make_rlist_command_includes_log_debug():
insert_logging_mock(logging.DEBUG)
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -185,7 +185,7 @@ def test_make_rlist_command_includes_json_but_not_debug():
insert_logging_mock(logging.DEBUG)
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',))
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -203,7 +203,7 @@ def test_make_rlist_command_includes_json_but_not_debug():
def test_make_rlist_command_includes_json():
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',))
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -223,7 +223,7 @@ def test_make_rlist_command_includes_lock_wait():
('--lock-wait', '5')
).and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -241,7 +241,7 @@ def test_make_rlist_command_includes_lock_wait():
def test_make_rlist_command_includes_local_path():
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -262,7 +262,7 @@ def test_make_rlist_command_includes_remote_path():
('--remote-path', 'borg2')
).and_return(()).and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -283,7 +283,7 @@ def test_make_rlist_command_transforms_prefix_into_match_archives():
('--match-archives', 'sh:foo*')
)
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -319,7 +319,7 @@ def test_make_rlist_command_prefers_prefix_over_archive_name_format():
def test_make_rlist_command_transforms_archive_name_format_into_match_archives():
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
'bar-{now}', '1.2.3' # noqa: FS003
None, 'bar-{now}', '1.2.3' # noqa: FS003
).and_return(('--match-archives', 'sh:bar-*'))
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -337,7 +337,7 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives()
def test_make_rlist_command_includes_short():
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--short',))
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -368,7 +368,7 @@ def test_make_rlist_command_includes_short():
def test_make_rlist_command_includes_additional_flags(argument_name):
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, '1.2.3'
None, None, '1.2.3'
).and_return(())
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(
(f"--{argument_name.replace('_', '-')}", 'value')

View File

@ -185,7 +185,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
'bar-{now}', '2.3.4' # noqa: FS003
None, 'bar-{now}', '2.3.4' # noqa: FS003
).and_return(('--match-archives', 'sh:bar-*'))
flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))