From c64d0100d59038dc1af6a40cd7a5aec83769b257 Mon Sep 17 00:00:00 2001 From: Nick Whyte Date: Sun, 4 Mar 2018 17:17:39 +1100 Subject: [PATCH] Only check archives with matching prefix. --- AUTHORS | 1 + borgmatic/borg/check.py | 5 ++- borgmatic/config/schema.yaml | 7 ++++ borgmatic/config/validate.py | 7 ++++ borgmatic/tests/unit/borg/test_check.py | 39 ++++++++++++++++---- borgmatic/tests/unit/config/test_validate.py | 32 +++++++++++++++- borgmatic/tests/unit/test_verbosity.py | 4 +- borgmatic/verbosity.py | 2 +- 8 files changed, 84 insertions(+), 13 deletions(-) diff --git a/AUTHORS b/AUTHORS index abf17f979..2bf82b39a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -8,3 +8,4 @@ newtonne: Read encryption password from external file Robin `ypid` Schneider: Support additional options of Borg Scott Squires: Custom archive names Thomas LÉVEIL: Support for a keep_minutely prune option +Nick Whyte: Support prefix filtering for archive consistency checks diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 04b2daace..01bfafb9b 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -82,10 +82,13 @@ def check_archives(verbosity, repository, storage_config, consistency_config, lo VERBOSITY_LOTS: ('--debug',), }.get(verbosity, ()) + prefix = consistency_config.get('prefix', '{hostname}-') + prefix_flags = ('--prefix', prefix) if prefix else () + full_command = ( local_path, 'check', repository, - ) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags + ) + _make_check_flags(checks, check_last) + prefix_flags + remote_path_flags + lock_wait_flags + verbosity_flags # The check command spews to stdout/stderr even without the verbose flag. Suppress it. stdout = None if verbosity_flags else open(os.devnull, 'w') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 791e3c143..770c1f52f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -218,6 +218,13 @@ map: desc: Restrict the number of checked archives to the last n. Applies only to the "archives" check. example: 3 + prefix: + type: scalar + desc: | + When performing consistency checks, only consider archive names starting with + this prefix. Borg placeholders can be used. See the output of + "borg help placeholders" for details. Default is "{hostname}-". + example: sourcehostname hooks: desc: | Shell commands or scripts to execute before and after a backup or if an error has occurred. diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 1ce39247e..2645e5521 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -8,6 +8,8 @@ import pykwalify.errors from ruamel import yaml +logger = logging.getLogger(__name__) + def schema_filename(): ''' Path to the installed YAML configuration schema file, used to validate and parse the @@ -50,6 +52,11 @@ def apply_logical_validation(config_filename, parsed_configuration): ) ) + consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix') + if archive_name_format and not consistency_prefix: + logger.warning('Since version 1.2.0, if you provide `archive_name_format`, you must also' + ' specify `consistency.prefix`.') + def parse_configuration(config_filename, schema_filename): ''' diff --git a/borgmatic/tests/unit/borg/test_check.py b/borgmatic/tests/unit/borg/test_check.py index 361a720ec..1f97ea510 100644 --- a/borgmatic/tests/unit/borg/test_check.py +++ b/borgmatic/tests/unit/borg/test_check.py @@ -89,7 +89,7 @@ def test_make_check_flags_with_default_checks_and_last_returns_last_flag(): ) def test_check_archives_calls_borg_with_parameters(checks): check_last = flexmock() - consistency_config = flexmock().should_receive('get').and_return(check_last).mock + consistency_config = {'check_last': check_last} flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) stdout = flexmock() @@ -111,7 +111,7 @@ def test_check_archives_calls_borg_with_parameters(checks): def test_check_archives_with_extract_check_calls_extract_only(): checks = ('extract',) check_last = flexmock() - consistency_config = flexmock().should_receive('get').and_return(check_last).mock + consistency_config = {'check_last': check_last} flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').never() flexmock(module.extract).should_receive('extract_last_archive_dry_run').once() @@ -127,7 +127,7 @@ def test_check_archives_with_extract_check_calls_extract_only(): def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter(): checks = ('repository',) - consistency_config = flexmock().should_receive('get').and_return(None).mock + consistency_config = {'check_last': None} flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( @@ -145,7 +145,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter(): def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter(): checks = ('repository',) - consistency_config = flexmock().should_receive('get').and_return(None).mock + consistency_config = {'check_last': None} flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( @@ -162,7 +162,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter(): def test_check_archives_without_any_checks_bails(): - consistency_config = flexmock().should_receive('get').and_return(None).mock + consistency_config = {'check_last': None} flexmock(module).should_receive('_parse_checks').and_return(()) insert_subprocess_never() @@ -177,7 +177,7 @@ def test_check_archives_without_any_checks_bails(): def test_check_archives_with_local_path_calls_borg_via_local_path(): checks = ('repository',) check_last = flexmock() - consistency_config = flexmock().should_receive('get').and_return(check_last).mock + consistency_config = {'check_last': check_last} flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) stdout = flexmock() @@ -200,7 +200,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(): checks = ('repository',) check_last = flexmock() - consistency_config = flexmock().should_receive('get').and_return(check_last).mock + consistency_config = {'check_last': check_last} flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) stdout = flexmock() @@ -223,7 +223,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): checks = ('repository',) check_last = flexmock() - consistency_config = flexmock().should_receive('get').and_return(check_last).mock + consistency_config = {'check_last': check_last} flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) stdout = flexmock() @@ -240,3 +240,26 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): storage_config={'lock_wait': 5}, consistency_config=consistency_config, ) + + +def test_check_archives_with_retention_prefix(): + checks = ('repository',) + check_last = flexmock() + consistency_config = {'check_last': check_last, 'prefix': 'foo-'} + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) + stdout = flexmock() + insert_subprocess_mock( + ('borg', 'check', 'repo', '--prefix', 'foo-'), + stdout=stdout, stderr=STDOUT, + ) + + flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) + flexmock(module.os).should_receive('devnull') + + module.check_archives( + verbosity=None, + repository='repo', + storage_config={}, + consistency_config=consistency_config, + ) diff --git a/borgmatic/tests/unit/config/test_validate.py b/borgmatic/tests/unit/config/test_validate.py index 9a943a035..46e62477f 100644 --- a/borgmatic/tests/unit/config/test_validate.py +++ b/borgmatic/tests/unit/config/test_validate.py @@ -1,4 +1,5 @@ import pytest +from flexmock import flexmock from borgmatic.config import validate as module @@ -23,13 +24,42 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_ }, ) +def test_apply_logical_validation_raises_if_archive_name_format_present_without_retention_prefix(): + with pytest.raises(module.Validation_error): + module.apply_logical_validation( + 'config.yaml', + { + 'storage': {'archive_name_format': '{hostname}-{now}'}, + 'retention': {'keep_daily': 7}, + 'consistency': {'prefix': '{hostname}-'} + }, + ) + + +def test_apply_logical_validation_warns_if_archive_name_format_present_without_consistency_prefix(): + logger = flexmock(module.logger) + logger.should_receive('warning').once() -def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present(): module.apply_logical_validation( 'config.yaml', { 'storage': {'archive_name_format': '{hostname}-{now}'}, 'retention': {'prefix': '{hostname}-'}, + 'consistency': {}, + }, + ) + + +def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present(): + logger = flexmock(module.logger) + logger.should_receive('warning').never() + + module.apply_logical_validation( + 'config.yaml', + { + 'storage': {'archive_name_format': '{hostname}-{now}'}, + 'retention': {'prefix': '{hostname}-'}, + 'consistency': {'prefix': '{hostname}-'} }, ) diff --git a/borgmatic/tests/unit/test_verbosity.py b/borgmatic/tests/unit/test_verbosity.py index 685aed07a..f62eec27f 100644 --- a/borgmatic/tests/unit/test_verbosity.py +++ b/borgmatic/tests/unit/test_verbosity.py @@ -7,5 +7,5 @@ def test_verbosity_to_log_level_maps_known_verbosity_to_log_level(): assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO -def test_verbosity_to_log_level_maps_unknown_verbosity_to_error_level(): - assert module.verbosity_to_log_level('my pants') == logging.ERROR +def test_verbosity_to_log_level_maps_unknown_verbosity_to_warning_level(): + assert module.verbosity_to_log_level('my pants') == logging.WARNING diff --git a/borgmatic/verbosity.py b/borgmatic/verbosity.py index e1ea796dd..91264e4aa 100644 --- a/borgmatic/verbosity.py +++ b/borgmatic/verbosity.py @@ -12,4 +12,4 @@ def verbosity_to_log_level(verbosity): return { VERBOSITY_SOME: logging.INFO, VERBOSITY_LOTS: logging.DEBUG, - }.get(verbosity, logging.ERROR) + }.get(verbosity, logging.WARNING)