Add glob ("*") support to the "--repository" flag (#898).
All checks were successful
build / test (push) Successful in 5m49s
build / docs (push) Successful in 1m38s

This commit is contained in:
Dan Helfman 2024-08-20 12:49:50 -07:00
parent 548aceb3d5
commit c5633227bf
4 changed files with 119 additions and 76 deletions

2
NEWS
View File

@ -1,5 +1,7 @@
1.8.14.dev0
* #896: Fix an error in borgmatic rcreate/init on an empty repository directory with Borg 1.4.
* #898: Add glob ("*") support to the "--repository" flag. Just quote any values containing
globs so your shell doesn't interpret them.
* #899: Fix for a "bad character" Borg error in which the "spot" check fed Borg an invalid pattern.
* #900: Fix for a potential traceback (TypeError) during the handling of another error.
* #904: Clarify the configuration reference about the "spot" check options:

View File

@ -425,7 +425,7 @@ def make_parsers():
)
rcreate_group.add_argument(
'--repository',
help='Path of the new repository to create (must be already specified in a borgmatic configuration file), defaults to the configured repository if there is only one',
help='Path of the new repository to create (must be already specified in a borgmatic configuration file), defaults to the configured repository if there is only one, quoted globs supported',
)
rcreate_group.add_argument(
'--copy-crypt-key',
@ -460,7 +460,7 @@ def make_parsers():
transfer_group = transfer_parser.add_argument_group('transfer arguments')
transfer_group.add_argument(
'--repository',
help='Path of existing destination repository to transfer archives to, defaults to the configured repository if there is only one',
help='Path of existing destination repository to transfer archives to, defaults to the configured repository if there is only one, quoted globs supported',
)
transfer_group.add_argument(
'--source-repository',
@ -533,7 +533,7 @@ def make_parsers():
prune_group = prune_parser.add_argument_group('prune arguments')
prune_group.add_argument(
'--repository',
help='Path of specific existing repository to prune (must be already specified in a borgmatic configuration file)',
help='Path of specific existing repository to prune (must be already specified in a borgmatic configuration file), quoted globs supported',
)
prune_group.add_argument(
'--stats',
@ -577,7 +577,7 @@ def make_parsers():
compact_group = compact_parser.add_argument_group('compact arguments')
compact_group.add_argument(
'--repository',
help='Path of specific existing repository to compact (must be already specified in a borgmatic configuration file)',
help='Path of specific existing repository to compact (must be already specified in a borgmatic configuration file), quoted globs supported',
)
compact_group.add_argument(
'--progress',
@ -613,7 +613,7 @@ def make_parsers():
create_group = create_parser.add_argument_group('create arguments')
create_group.add_argument(
'--repository',
help='Path of specific existing repository to backup to (must be already specified in a borgmatic configuration file)',
help='Path of specific existing repository to backup to (must be already specified in a borgmatic configuration file), quoted globs supported',
)
create_group.add_argument(
'--progress',
@ -647,7 +647,7 @@ def make_parsers():
check_group = check_parser.add_argument_group('check arguments')
check_group.add_argument(
'--repository',
help='Path of specific existing repository to check (must be already specified in a borgmatic configuration file)',
help='Path of specific existing repository to check (must be already specified in a borgmatic configuration file), quoted globs supported',
)
check_group.add_argument(
'--progress',
@ -701,7 +701,7 @@ def make_parsers():
delete_group = delete_parser.add_argument_group('delete arguments')
delete_group.add_argument(
'--repository',
help='Path of repository to delete or delete archives from, defaults to the configured repository if there is only one',
help='Path of repository to delete or delete archives from, defaults to the configured repository if there is only one, quoted globs supported',
)
delete_group.add_argument(
'--archive',
@ -792,7 +792,7 @@ def make_parsers():
extract_group = extract_parser.add_argument_group('extract arguments')
extract_group.add_argument(
'--repository',
help='Path of repository to extract, defaults to the configured repository if there is only one',
help='Path of repository to extract, defaults to the configured repository if there is only one, quoted globs supported',
)
extract_group.add_argument(
'--archive', help='Name of archive to extract (or "latest")', required=True
@ -854,7 +854,7 @@ def make_parsers():
)
config_bootstrap_group.add_argument(
'--repository',
help='Path of repository to extract config files from',
help='Path of repository to extract config files from, quoted globs supported',
required=True,
)
config_bootstrap_group.add_argument(
@ -952,7 +952,7 @@ def make_parsers():
export_tar_group = export_tar_parser.add_argument_group('export-tar arguments')
export_tar_group.add_argument(
'--repository',
help='Path of repository to export from, defaults to the configured repository if there is only one',
help='Path of repository to export from, defaults to the configured repository if there is only one, quoted globs supported',
)
export_tar_group.add_argument(
'--archive', help='Name of archive to export (or "latest")', required=True
@ -998,7 +998,7 @@ def make_parsers():
mount_group = mount_parser.add_argument_group('mount arguments')
mount_group.add_argument(
'--repository',
help='Path of repository to use, defaults to the configured repository if there is only one',
help='Path of repository to use, defaults to the configured repository if there is only one, quoted globs supported',
)
mount_group.add_argument('--archive', help='Name of archive to mount (or "latest")')
mount_group.add_argument(
@ -1080,7 +1080,7 @@ def make_parsers():
rdelete_group = rdelete_parser.add_argument_group('delete arguments')
rdelete_group.add_argument(
'--repository',
help='Path of repository to delete, defaults to the configured repository if there is only one',
help='Path of repository to delete, defaults to the configured repository if there is only one, quoted globs supported',
)
rdelete_group.add_argument(
'--list',
@ -1117,7 +1117,7 @@ def make_parsers():
restore_group = restore_parser.add_argument_group('restore arguments')
restore_group.add_argument(
'--repository',
help='Path of repository to restore from, defaults to the configured repository if there is only one',
help='Path of repository to restore from, defaults to the configured repository if there is only one, quoted globs supported',
)
restore_group.add_argument(
'--archive', help='Name of archive to restore from (or "latest")', required=True
@ -1171,7 +1171,7 @@ def make_parsers():
rlist_group = rlist_parser.add_argument_group('rlist arguments')
rlist_group.add_argument(
'--repository',
help='Path of repository to list, defaults to the configured repositories',
help='Path of repository to list, defaults to the configured repositories, quoted globs supported',
)
rlist_group.add_argument(
'--short', default=False, action='store_true', help='Output only archive names'
@ -1231,7 +1231,7 @@ def make_parsers():
list_group = list_parser.add_argument_group('list arguments')
list_group.add_argument(
'--repository',
help='Path of repository containing archive to list, defaults to the configured repositories',
help='Path of repository containing archive to list, defaults to the configured repositories, quoted globs supported',
)
list_group.add_argument('--archive', help='Name of the archive to list (or "latest")')
list_group.add_argument(
@ -1298,7 +1298,7 @@ def make_parsers():
rinfo_group = rinfo_parser.add_argument_group('rinfo arguments')
rinfo_group.add_argument(
'--repository',
help='Path of repository to show info for, defaults to the configured repository if there is only one',
help='Path of repository to show info for, defaults to the configured repository if there is only one, quoted globs supported',
)
rinfo_group.add_argument(
'--json', dest='json', default=False, action='store_true', help='Output results as JSON'
@ -1315,7 +1315,7 @@ def make_parsers():
info_group = info_parser.add_argument_group('info arguments')
info_group.add_argument(
'--repository',
help='Path of repository containing archive to show info for, defaults to the configured repository if there is only one',
help='Path of repository containing archive to show info for, defaults to the configured repository if there is only one, quoted globs supported',
)
info_group.add_argument('--archive', help='Name of archive to show info for (or "latest")')
info_group.add_argument(
@ -1376,7 +1376,7 @@ def make_parsers():
break_lock_group = break_lock_parser.add_argument_group('break-lock arguments')
break_lock_group.add_argument(
'--repository',
help='Path of repository to break the lock for, defaults to the configured repository if there is only one',
help='Path of repository to break the lock for, defaults to the configured repository if there is only one, quoted globs supported',
)
break_lock_group.add_argument(
'-h', '--help', action='help', help='Show this help message and exit'
@ -1416,7 +1416,7 @@ def make_parsers():
)
key_export_group.add_argument(
'--repository',
help='Path of repository to export the key for, defaults to the configured repository if there is only one',
help='Path of repository to export the key for, defaults to the configured repository if there is only one, quoted globs supported',
)
key_export_group.add_argument(
'--path',
@ -1437,7 +1437,7 @@ def make_parsers():
borg_group = borg_parser.add_argument_group('borg arguments')
borg_group.add_argument(
'--repository',
help='Path of repository to pass to Borg, defaults to the configured repositories',
help='Path of repository to pass to Borg, defaults to the configured repositories, quoted globs supported',
)
borg_group.add_argument('--archive', help='Name of archive to pass to Borg (or "latest")')
borg_group.add_argument(

View File

@ -1,3 +1,4 @@
import fnmatch
import os
import jsonschema
@ -149,18 +150,30 @@ def normalize_repository_path(repository):
return repository
def glob_match(first, second):
'''
Given two strings, return whether the first matches the second. Globs are
supported.
'''
if first is None or second is None:
return False
return fnmatch.fnmatch(first, second) or fnmatch.fnmatch(second, first)
def repositories_match(first, second):
'''
Given two repository dicts with keys 'path' (relative and/or absolute),
and 'label', or two repository paths, return whether they match.
Given two repository dicts with keys "path" (relative and/or absolute),
and "label", two repository paths as strings, or a mix of the two formats,
return whether they match. Globs are supported.
'''
if isinstance(first, str):
first = {'path': first, 'label': first}
if isinstance(second, str):
second = {'path': second, 'label': second}
return (first.get('label') == second.get('label')) or (
normalize_repository_path(first.get('path'))
== normalize_repository_path(second.get('path'))
return glob_match(first.get('label'), second.get('label')) or glob_match(
normalize_repository_path(first.get('path')), normalize_repository_path(second.get('path'))
)

View File

@ -62,7 +62,7 @@ def test_validation_error_string_contains_errors():
def test_apply_logical_validation_raises_if_unknown_repository_in_check_repositories():
flexmock(module).format_json_error = lambda error: error.message
flexmock(module).should_receive('repositories_match').and_return(False)
with pytest.raises(module.Validation_error):
module.apply_logical_validation(
@ -75,7 +75,9 @@ def test_apply_logical_validation_raises_if_unknown_repository_in_check_reposito
)
def test_apply_logical_validation_does_not_raise_if_known_repository_path_in_check_repositories():
def test_apply_logical_validation_does_not_raise_if_known_repository_in_check_repositories():
flexmock(module).should_receive('repositories_match').and_return(True)
module.apply_logical_validation(
'config.yaml',
{
@ -86,35 +88,6 @@ def test_apply_logical_validation_does_not_raise_if_known_repository_path_in_che
)
def test_apply_logical_validation_does_not_raise_if_known_repository_label_in_check_repositories():
module.apply_logical_validation(
'config.yaml',
{
'repositories': [
{'path': 'repo.borg', 'label': 'my_repo'},
{'path': 'other.borg', 'label': 'other_repo'},
],
'keep_secondly': 1000,
'check_repositories': ['my_repo'],
},
)
def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present():
module.apply_logical_validation(
'config.yaml',
{
'archive_name_format': '{hostname}-{now}', # noqa: FS003
'prefix': '{hostname}-', # noqa: FS003
'prefix': '{hostname}-', # noqa: FS003
},
)
def test_apply_logical_validation_does_not_raise_otherwise():
module.apply_logical_validation('config.yaml', {'keep_secondly': 1000})
def test_normalize_repository_path_passes_through_remote_repository():
repository = 'example.org:test.borg'
@ -143,39 +116,94 @@ def test_normalize_repository_path_resolves_relative_repository():
module.normalize_repository_path(repository) == absolute
def test_repositories_match_does_not_raise():
@pytest.mark.parametrize(
'first,second,expected_result',
(
(None, None, False),
('foo', None, False),
(None, 'bar', False),
('foo', 'foo', True),
('foo', 'bar', False),
('foo*', 'foof', True),
('barf', 'bar*', True),
('foo*', 'bar*', False),
),
)
def test_glob_match_matches_globs(first, second, expected_result):
assert module.glob_match(first=first, second=second) is expected_result
def test_repositories_match_matches_on_path():
flexmock(module).should_receive('normalize_repository_path')
module.repositories_match('foo', 'bar')
def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config():
flexmock(module).should_receive('repositories_match').replace_with(
flexmock(module).should_receive('glob_match').replace_with(
lambda first, second: first == second
)
module.guard_configuration_contains_repository(
repository='repo', configurations={'config.yaml': {'repositories': ['repo']}}
module.repositories_match(
{'path': 'foo', 'label': 'my repo'}, {'path': 'foo', 'label': 'other repo'}
) is True
def test_repositories_match_matches_on_label():
flexmock(module).should_receive('normalize_repository_path')
flexmock(module).should_receive('glob_match').replace_with(
lambda first, second: first == second
)
module.repositories_match(
{'path': 'foo', 'label': 'my repo'}, {'path': 'bar', 'label': 'my repo'}
) is True
def test_repositories_match_with_different_paths_and_labels_does_not_match():
flexmock(module).should_receive('normalize_repository_path')
flexmock(module).should_receive('glob_match').replace_with(
lambda first, second: first == second
)
module.repositories_match(
{'path': 'foo', 'label': 'my repo'}, {'path': 'bar', 'label': 'other repo'}
) is False
def test_repositories_match_matches_on_string_repository():
flexmock(module).should_receive('normalize_repository_path')
flexmock(module).should_receive('glob_match').replace_with(
lambda first, second: first == second
)
module.repositories_match('foo', 'foo') is True
def test_repositories_match_with_different_string_repositories_does_not_match():
flexmock(module).should_receive('normalize_repository_path')
flexmock(module).should_receive('glob_match').replace_with(
lambda first, second: first == second
)
module.repositories_match('foo', 'bar') is False
def test_repositories_match_supports_mixed_repositories():
flexmock(module).should_receive('normalize_repository_path')
flexmock(module).should_receive('glob_match').replace_with(
lambda first, second: first == second
)
module.repositories_match({'path': 'foo', 'label': 'my foo'}, 'bar') is False
def test_guard_configuration_contains_repository_does_not_raise_when_repository_matches():
flexmock(module).should_receive('repositories_match').and_return(True)
def test_guard_configuration_contains_repository_does_not_raise_when_repository_label_in_config():
module.guard_configuration_contains_repository(
repository='repo',
configurations={'config.yaml': {'repositories': [{'path': 'foo/bar', 'label': 'repo'}]}},
)
def test_guard_configuration_contains_repository_does_not_raise_when_repository_not_given():
module.guard_configuration_contains_repository(
repository=None, configurations={'config.yaml': {'repositories': ['repo']}}
)
def test_guard_configuration_contains_repository_errors_when_repository_missing_from_config():
flexmock(module).should_receive('repositories_match').replace_with(
lambda first, second: first == second
)
def test_guard_configuration_contains_repository_errors_when_repository_does_not_match():
flexmock(module).should_receive('repositories_match').and_return(False)
with pytest.raises(ValueError):
module.guard_configuration_contains_repository(