From fa38de2de7d625d89e3753c84e576873b9341a20 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 13 Oct 2018 20:34:51 -0700 Subject: [PATCH] Enable consistency checks for only certain repositories via "check_repositories" (#73). --- NEWS | 2 ++ borgmatic/commands/borgmatic.py | 6 +++--- borgmatic/config/checks.py | 9 +++++++++ borgmatic/config/schema.yaml | 10 ++++++++++ borgmatic/config/validate.py | 13 +++++++++++++ tests/unit/config/test_checks.py | 23 +++++++++++++++++++++++ tests/unit/config/test_validate.py | 23 +++++++++++++++++++++++ 7 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 borgmatic/config/checks.py create mode 100644 tests/unit/config/test_checks.py diff --git a/NEWS b/NEWS index e83a767de..fa055448b 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.2.8.dev0 + * #73: Enable consistency checks for only certain repositories via "check_repositories" option in + borgmatic's consistency configuration. Handy for large repositories that take forever to check. * Include link to issue tracker within various command output. * Run continuous integration tests on a matrix of Python and Borg versions. diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 3e418779a..979cf3600 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -13,7 +13,7 @@ from borgmatic.borg import ( info as borg_info, ) from borgmatic.commands import hook -from borgmatic.config import collect, convert, validate +from borgmatic.config import checks, collect, convert, validate from borgmatic.signals import configure_signals from borgmatic.verbosity import verbosity_to_log_level @@ -147,7 +147,7 @@ def run_configuration(config_filename, args): # pragma: no cover _run_commands( args=args, consistency=consistency, - local_pagh=local_path, + local_path=local_path, location=location, remote_path=remote_path, retention=retention, @@ -213,7 +213,7 @@ def _run_commands_on_repository( local_path=local_path, remote_path=remote_path, ) - if args.check: + if args.check and checks.repository_enabled_for_checks(repository, consistency): logger.info('{}: Running consistency checks'.format(repository)) borg_check.check_archives( repository, storage, consistency, local_path=local_path, remote_path=remote_path diff --git a/borgmatic/config/checks.py b/borgmatic/config/checks.py new file mode 100644 index 000000000..13361ea14 --- /dev/null +++ b/borgmatic/config/checks.py @@ -0,0 +1,9 @@ +def repository_enabled_for_checks(repository, consistency): + ''' + Given a repository name and a consistency configuration dict, return whether the repository + is enabled to have consistency checks run. + ''' + if not consistency.get('check_repositories'): + return True + + return repository in consistency['check_repositories'] diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index f3ca6a02c..193353678 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -236,6 +236,16 @@ map: example: - repository - archives + check_repositories: + seq: + - type: scalar + desc: | + Paths to a subset of the repositories in the location section on which to run + consistency checks. Handy in case some of your repositories are very large, and + so running consistency checks on them would take too long. Defaults to running + consistency checks on all repositories configured in the location section. + example: + - user@backupserver:sourcehostname.borg check_last: type: int desc: Restrict the number of checked archives to the last n. Applies only to the diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index a5133f461..5c0b32ba2 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -51,6 +51,19 @@ def apply_logical_validation(config_filename, parsed_configuration): ('If you provide an archive_name_format, you must also specify a retention prefix.',), ) + location_repositories = parsed_configuration.get('location', {}).get('repositories') + check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) + for repository in check_repositories: + if repository not in location_repositories: + raise Validation_error( + config_filename, + ( + 'Unknown repository in the consistency section\'s check_repositories: {}'.format( + repository + ), + ), + ) + consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix') if archive_name_format and not consistency_prefix: logger.warning( diff --git a/tests/unit/config/test_checks.py b/tests/unit/config/test_checks.py new file mode 100644 index 000000000..df6df1eca --- /dev/null +++ b/tests/unit/config/test_checks.py @@ -0,0 +1,23 @@ +from borgmatic.config import checks as module + + +def test_repository_enabled_for_checks_defaults_to_enabled_for_all_repositories(): + enabled = module.repository_enabled_for_checks('repo.borg', consistency={}) + + assert enabled + + +def test_repository_enabled_for_checks_is_enabled_for_specified_repositories(): + enabled = module.repository_enabled_for_checks( + 'repo.borg', consistency={'check_repositories': ['repo.borg', 'other.borg']} + ) + + assert enabled + + +def test_repository_enabled_for_checks_is_disabled_for_other_repositories(): + enabled = module.repository_enabled_for_checks( + 'repo.borg', consistency={'check_repositories': ['other.borg']} + ) + + assert not enabled diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index eb98b14c5..a542e21f3 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -51,6 +51,29 @@ def test_apply_logical_validation_warns_if_archive_name_format_present_without_c ) +def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories(): + with pytest.raises(module.Validation_error): + module.apply_logical_validation( + 'config.yaml', + { + 'location': {'repositories': ['repo.borg', 'other.borg']}, + 'retention': {'keep_secondly': 1000}, + 'consistency': {'check_repositories': ['repo.borg', 'unknown.borg']}, + }, + ) + + +def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_repositories(): + module.apply_logical_validation( + 'config.yaml', + { + 'location': {'repositories': ['repo.borg', 'other.borg']}, + 'retention': {'keep_secondly': 1000}, + 'consistency': {'check_repositories': ['repo.borg']}, + }, + ) + + def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present(): logger = flexmock(module.logger) logger.should_receive('warning').never()