diff --git a/NEWS b/NEWS index 3b710caa..e9eb1c6b 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,8 @@ support empty sections without erroring. * #774: Disallow the "--dry-run" flag with the "borg" action, as borgmatic can't guarantee the Borg command won't have side effects. + * #779: Add a "--match-archives" flag to the "check" action for selecting the archives to check, + overriding the existing "archive_name_format" and "match_archives" options in configuration. 1.8.3 * #665: BREAKING: Simplify logging logic as follows: Syslog verbosity is now disabled by diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index fe800156..79e20239 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -39,13 +39,10 @@ def run_check( repository['path'], config, local_borg_version, + check_arguments, global_arguments, local_path=local_path, remote_path=remote_path, - progress=check_arguments.progress, - repair=check_arguments.repair, - only_checks=check_arguments.only, - force=check_arguments.force, ) borgmatic.hooks.command.execute_hook( config.get('after_check'), diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index eabbdd31..14d7a827 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -149,11 +149,13 @@ def filter_checks_on_frequency( return tuple(filtered_checks) -def make_archive_filter_flags(local_borg_version, config, checks, check_last=None, prefix=None): +def make_archive_filter_flags( + local_borg_version, config, checks, check_arguments, check_last=None, prefix=None +): ''' - Given the local Borg version, a configuration dict, a parsed sequence of checks, the check last - value, and a consistency check prefix, transform the checks into tuple of command-line flags for - filtering archives in a check command. + Given the local Borg version, a configuration dict, a parsed sequence of checks, check arguments + as an argparse.Namespace instance, the check last value, and a consistency check prefix, + transform the checks into tuple of command-line flags for filtering archives in a check command. If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if a prefix value is given and "archives" is in checks, then include a "--match-archives" flag. @@ -168,7 +170,7 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_last=Non if prefix else ( flags.make_match_archives_flags( - config.get('match_archives'), + check_arguments.match_archives or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -353,18 +355,15 @@ def check_archives( repository_path, config, local_borg_version, + check_arguments, global_arguments, local_path='borg', remote_path=None, - progress=None, - repair=None, - only_checks=None, - force=None, ): ''' - Given a local or remote repository path, a configuration dict, local/remote commands to run, - whether to include progress information, whether to attempt a repair, and an optional list of - checks to use instead of configured checks, check the contained Borg archives for consistency. + Given a local or remote repository path, a configuration dict, the local Borg version, check + arguments as an argparse.Namespace instance, global arguments, and local/remote commands to run, + check the contained Borg archives for consistency. If there are no consistency checks to run, skip running them. @@ -389,11 +388,11 @@ def check_archives( check_last = config.get('check_last', None) prefix = config.get('prefix') - configured_checks = parse_checks(config, only_checks) + configured_checks = parse_checks(config, check_arguments.only_checks) lock_wait = None extra_borg_options = config.get('extra_borg_options', {}).get('check', '') archive_filter_flags = make_archive_filter_flags( - local_borg_version, config, configured_checks, check_last, prefix + local_borg_version, config, configured_checks, check_arguments, check_last, prefix ) archives_check_id = make_archives_check_id(archive_filter_flags) @@ -401,7 +400,7 @@ def check_archives( config, borg_repository_id, configured_checks, - force, + check_arguments.force, archives_check_id, ) @@ -416,13 +415,13 @@ def check_archives( full_command = ( (local_path, 'check') - + (('--repair',) if repair else ()) + + (('--repair',) if check_arguments.repair else ()) + make_check_flags(checks, archive_filter_flags) + (('--remote-path', remote_path) if remote_path else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags - + (('--progress',) if progress else ()) + + (('--progress',) if check_arguments.progress else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) @@ -431,7 +430,7 @@ def check_archives( # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. - if repair or progress: + if check_arguments.repair or check_arguments.progress: execute_command( full_command, output_file=DO_NOT_CAPTURE, extra_environment=borg_environment ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index caf25abd..c2b87ebc 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -604,11 +604,18 @@ def make_parsers(): action='store_true', help='Attempt to repair any inconsistencies found (for interactive use)', ) + check_group.add_argument( + '-a', + '--match-archives', + '--glob-archives', + metavar='PATTERN', + help='Only check archives with names matching this pattern', + ) check_group.add_argument( '--only', metavar='CHECK', choices=('repository', 'archives', 'data', 'extract'), - dest='only', + dest='only_checks', action='append', help='Run a particular consistency check (repository, archives, data, or extract) instead of configured checks (subject to configured frequency, can specify flag multiple times)', ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index c524b47e..8d8ac45d 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -201,6 +201,7 @@ def test_make_archive_filter_flags_with_default_checks_and_prefix_returns_defaul '1.2.3', {}, ('repository', 'archives'), + check_arguments=flexmock(match_archives=None), prefix='foo', ) @@ -215,6 +216,7 @@ def test_make_archive_filter_flags_with_all_checks_and_prefix_returns_default_fl '1.2.3', {}, ('repository', 'archives', 'extract'), + check_arguments=flexmock(match_archives=None), prefix='foo', ) @@ -229,6 +231,7 @@ def test_make_archive_filter_flags_with_all_checks_and_prefix_without_borg_featu '1.2.3', {}, ('repository', 'archives', 'extract'), + check_arguments=flexmock(match_archives=None), prefix='foo', ) @@ -239,7 +242,9 @@ def test_make_archive_filter_flags_with_archives_check_and_last_includes_last_fl flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), check_last=3) + flags = module.make_archive_filter_flags( + '1.2.3', {}, ('archives',), check_arguments=flexmock(match_archives=None), check_last=3 + ) assert flags == ('--last', '3') @@ -248,7 +253,9 @@ def test_make_archive_filter_flags_with_data_check_and_last_includes_last_flag() flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), check_last=3) + flags = module.make_archive_filter_flags( + '1.2.3', {}, ('data',), check_arguments=flexmock(match_archives=None), check_last=3 + ) assert flags == ('--last', '3') @@ -257,7 +264,9 @@ def test_make_archive_filter_flags_with_repository_check_and_last_omits_last_fla flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), check_last=3) + flags = module.make_archive_filter_flags( + '1.2.3', {}, ('repository',), check_arguments=flexmock(match_archives=None), check_last=3 + ) assert flags == () @@ -266,7 +275,13 @@ def test_make_archive_filter_flags_with_default_checks_and_last_includes_last_fl flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives'), + check_arguments=flexmock(match_archives=None), + check_last=3, + ) assert flags == ('--last', '3') @@ -275,7 +290,9 @@ def test_make_archive_filter_flags_with_archives_check_and_prefix_includes_match flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix='foo-') + flags = module.make_archive_filter_flags( + '1.2.3', {}, ('archives',), check_arguments=flexmock(match_archives=None), prefix='foo-' + ) assert flags == ('--match-archives', 'sh:foo-*') @@ -284,11 +301,30 @@ def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_arc flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), prefix='foo-') + flags = module.make_archive_filter_flags( + '1.2.3', {}, ('data',), check_arguments=flexmock(match_archives=None), prefix='foo-' + ) assert flags == ('--match-archives', 'sh:foo-*') +def test_make_archive_filter_flags_prefers_check_arguments_match_archives_to_config_match_archives(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + 'baz-*', None, '1.2.3' + ).and_return(('--match-archives', 'sh:baz-*')) + + flags = module.make_archive_filter_flags( + '1.2.3', + {'match_archives': 'bar-{now}'}, # noqa: FS003 + ('archives',), + check_arguments=flexmock(match_archives='baz-*'), + prefix='', + ) + + assert flags == ('--match-archives', 'sh:baz-*') + + def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( @@ -296,7 +332,11 @@ def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_arc ).and_return(('--match-archives', 'sh:bar-*')) flags = module.make_archive_filter_flags( - '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 + '1.2.3', + {'archive_name_format': 'bar-{now}'}, # noqa: FS003 + ('archives',), + check_arguments=flexmock(match_archives=None), + prefix='', ) assert flags == ('--match-archives', 'sh:bar-*') @@ -306,7 +346,9 @@ def test_make_archive_filter_flags_with_archives_check_and_none_prefix_omits_mat flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix=None) + flags = module.make_archive_filter_flags( + '1.2.3', {}, ('archives',), check_arguments=flexmock(match_archives=None), prefix=None + ) assert flags == () @@ -315,7 +357,9 @@ def test_make_archive_filter_flags_with_repository_check_and_prefix_omits_match_ flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), prefix='foo-') + flags = module.make_archive_filter_flags( + '1.2.3', {}, ('repository',), check_arguments=flexmock(match_archives=None), prefix='foo-' + ) assert flags == () @@ -324,7 +368,13 @@ def test_make_archive_filter_flags_with_default_checks_and_prefix_includes_match flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives'), + check_arguments=flexmock(match_archives=None), + prefix='foo-', + ) assert flags == ('--match-archives', 'sh:foo-*') @@ -607,7 +657,7 @@ def test_upgrade_check_times_renames_stale_temporary_check_path(): module.upgrade_check_times(flexmock(), flexmock()) -def test_check_archives_with_progress_calls_borg_with_progress_parameter(): +def test_check_archives_with_progress_passes_through_to_borg(): checks = ('repository',) config = {'check_last': None} flexmock(module.rinfo).should_receive('display_repository_info').and_return( @@ -634,12 +684,14 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=True, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), - progress=True, ) -def test_check_archives_with_repair_calls_borg_with_repair_parameter(): +def test_check_archives_with_repair_passes_through_to_borg(): checks = ('repository',) config = {'check_last': None} flexmock(module.rinfo).should_receive('display_repository_info').and_return( @@ -666,8 +718,10 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=True, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), - repair=True, ) @@ -701,6 +755,9 @@ def test_check_archives_calls_borg_with_parameters(checks): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), ) @@ -723,6 +780,9 @@ def test_check_archives_with_json_error_raises(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), ) @@ -743,6 +803,9 @@ def test_check_archives_with_missing_json_keys_raises(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), ) @@ -769,11 +832,14 @@ def test_check_archives_with_extract_check_calls_extract_only(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), ) -def test_check_archives_with_log_info_calls_borg_with_info_parameter(): +def test_check_archives_with_log_info_passes_through_to_borg(): checks = ('repository',) config = {'check_last': None} flexmock(module.rinfo).should_receive('display_repository_info').and_return( @@ -795,11 +861,14 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), ) -def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): +def test_check_archives_with_log_debug_passes_through_to_borg(): checks = ('repository',) config = {'check_last': None} flexmock(module.rinfo).should_receive('display_repository_info').and_return( @@ -821,6 +890,9 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), ) @@ -841,6 +913,9 @@ def test_check_archives_without_any_checks_bails(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), ) @@ -867,12 +942,15 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), local_path='borg1', ) -def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_check_archives_with_remote_path_passes_through_to_borg(): checks = ('repository',) check_last = flexmock() config = {'check_last': check_last} @@ -894,12 +972,15 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), remote_path='borg1', ) -def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): +def test_check_archives_with_log_json_passes_through_to_borg(): checks = ('repository',) check_last = flexmock() config = {'check_last': check_last} @@ -921,11 +1002,14 @@ def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=True), ) -def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_check_archives_with_lock_wait_passes_through_to_borg(): checks = ('repository',) check_last = flexmock() config = {'lock_wait': 5, 'check_last': check_last} @@ -947,6 +1031,9 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), ) @@ -974,11 +1061,14 @@ def test_check_archives_with_retention_prefix(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), global_arguments=flexmock(log_json=False), ) -def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): +def test_check_archives_with_extra_borg_options_passes_through_to_borg(): checks = ('repository',) config = {'check_last': None, 'extra_borg_options': {'check': '--extra --options'}} flexmock(module.rinfo).should_receive('display_repository_info').and_return( @@ -999,5 +1089,42 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): repository_path='repo', config=config, local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives=None + ), + global_arguments=flexmock(log_json=False), + ) + + +def test_check_archives_with_match_archives_passes_through_to_borg(): + checks = ('archives',) + config = {'check_last': None} + flexmock(module.rinfo).should_receive('display_repository_info').and_return( + '{"repository": {"id": "repo"}}' + ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return( + ('--match-archives', 'foo-*') + ) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').and_return(('--match-archives', 'foo-*')) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'check', '--match-archives', 'foo-*', 'repo'), + extra_environment=None, + ).once() + flexmock(module).should_receive('make_check_time_path') + flexmock(module).should_receive('write_check_time') + + module.check_archives( + repository_path='repo', + config=config, + local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, repair=None, only_checks=None, force=None, match_archives='foo-*' + ), global_arguments=flexmock(log_json=False), )