From 94b9ef56be7fa4e0681e005f82bc1d29fafb5b35 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 23 Jan 2020 13:41:37 -0800 Subject: [PATCH] Change "exclude_if_present" option to support multiple filenames, rather than just a single filename (#280). --- NEWS | 2 ++ borgmatic/borg/create.py | 8 +++++-- borgmatic/config/normalize.py | 10 +++++++++ borgmatic/config/schema.yaml | 8 ++++--- borgmatic/config/validate.py | 3 ++- tests/integration/config/test_validate.py | 25 +++++++++++++++++++++ tests/unit/borg/test_create.py | 11 +++++++-- tests/unit/config/test_normalize.py | 27 +++++++++++++++++++++++ 8 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 borgmatic/config/normalize.py create mode 100644 tests/unit/config/test_normalize.py diff --git a/NEWS b/NEWS index 09525b14b..53e53b827 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ 1.4.23.dev0 * #274: Add ~/.config/borgmatic.d as another configuration directory default. * #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag. + * #280: Change "exclude_if_present" option to support multiple filenames that indicate a directory + should be excluded from backups, rather than just a single filename. * For "create" and "prune" actions, no longer list files or show detailed stats at any verbosities by default. You can opt back in with "--files" or "--stats" flags. * For "list" and "info" actions, show repository names even at verbosity 0. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index cfbd7b6db..7a3d27499 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -88,8 +88,12 @@ def _make_exclude_flags(location_config, exclude_filename=None): ) ) caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else () - if_present = location_config.get('exclude_if_present') - if_present_flags = ('--exclude-if-present', if_present) if if_present else () + if_present_flags = tuple( + itertools.chain.from_iterable( + ('--exclude-if-present', if_present) + for if_present in location_config.get('exclude_if_present', ()) + ) + ) keep_exclude_tags_flags = ( ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else () ) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py new file mode 100644 index 000000000..05080bb25 --- /dev/null +++ b/borgmatic/config/normalize.py @@ -0,0 +1,10 @@ +def normalize(config): + ''' + Given a configuration dict, apply particular hard-coded rules to normalize its contents to + adhere to the configuration schema. + ''' + exclude_if_present = config.get('location', {}).get('exclude_if_present') + + # "Upgrade" exclude_if_present from a string to a list. + if isinstance(exclude_if_present, str): + config['location']['exclude_if_present'] = [exclude_if_present] diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index c58ebdeca..46fa728ae 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -121,11 +121,13 @@ map: http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false. example: true exclude_if_present: - type: str + seq: + - type: str desc: | - Exclude directories that contain a file with the given filename. Defaults to not + Exclude directories that contain a file with the given filenames. Defaults to not set. - example: .nobackup + example: + - .nobackup keep_exclude_tags: type: bool desc: | diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index b7d34a9f8..504e18185 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -6,7 +6,7 @@ import pykwalify.core import pykwalify.errors import ruamel.yaml -from borgmatic.config import load, override +from borgmatic.config import load, normalize, override def schema_filename(): @@ -104,6 +104,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None): raise Validation_error(config_filename, (str(error),)) override.apply_overrides(config, overrides) + normalize.normalize(config) validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema)) parsed_result = validator.validate(raise_exception=False) diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index cbd4f5baa..fbd2098c1 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -239,3 +239,28 @@ def test_parse_configuration_applies_overrides(): 'local_path': 'borg2', } } + + +def test_parse_configuration_applies_normalization(): + mock_config_and_schema( + ''' + location: + source_directories: + - /home + + repositories: + - hostname.borg + + exclude_if_present: .nobackup + ''' + ) + + result = module.parse_configuration('config.yaml', 'schema.yaml') + + assert result == { + 'location': { + 'source_directories': ['/home'], + 'repositories': ['hostname.borg'], + 'exclude_if_present': ['.nobackup'], + } + } diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 5e51a51e4..68543718d 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -145,9 +145,16 @@ def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config def test_make_exclude_flags_includes_exclude_if_present_when_in_config(): - exclude_flags = module._make_exclude_flags(location_config={'exclude_if_present': 'exclude_me'}) + exclude_flags = module._make_exclude_flags( + location_config={'exclude_if_present': ['exclude_me', 'also_me']} + ) - assert exclude_flags == ('--exclude-if-present', 'exclude_me') + assert exclude_flags == ( + '--exclude-if-present', + 'exclude_me', + '--exclude-if-present', + 'also_me', + ) def test_make_exclude_flags_includes_keep_exclude_tags_when_true_in_config(): diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py new file mode 100644 index 000000000..58e51059c --- /dev/null +++ b/tests/unit/config/test_normalize.py @@ -0,0 +1,27 @@ +import pytest + +from borgmatic.config import normalize as module + + +@pytest.mark.parametrize( + 'config,expected_config', + ( + ( + {'location': {'exclude_if_present': '.nobackup'}}, + {'location': {'exclude_if_present': ['.nobackup']}}, + ), + ( + {'location': {'exclude_if_present': ['.nobackup']}}, + {'location': {'exclude_if_present': ['.nobackup']}}, + ), + ( + {'location': {'source_directories': ['foo', 'bar']}}, + {'location': {'source_directories': ['foo', 'bar']}}, + ), + ({'storage': {'compression': 'yes_please'}}, {'storage': {'compression': 'yes_please'}}), + ), +) +def test_normalize_applies_hard_coded_normalization_to_config(config, expected_config): + module.normalize(config) + + assert config == expected_config