From 6ff186731254645de675362dd4c2202cf72e53d1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 6 Mar 2019 12:06:27 -0800 Subject: [PATCH] Configuration files includes and merging (#148). --- NEWS | 4 ++ borgmatic/config/load.py | 62 ++++++++++++++++ borgmatic/config/validate.py | 10 +-- docs/how-to/make-per-application-backups.md | 79 +++++++++++++++++++++ setup.py | 2 +- tests/integration/config/test_load.py | 40 +++++++++++ tests/integration/config/test_validate.py | 61 ++++++++++++++++ 7 files changed, 253 insertions(+), 5 deletions(-) create mode 100644 borgmatic/config/load.py create mode 100644 tests/integration/config/test_load.py diff --git a/NEWS b/NEWS index 9e9ea40d..16ba71e8 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.3.0 + * #148: Configuration file includes and merging via "!include" tag to support reuse of common + options across configuration files. + 1.2.18 * #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in borgmatic's location section. diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py new file mode 100644 index 00000000..3ebe2cf1 --- /dev/null +++ b/borgmatic/config/load.py @@ -0,0 +1,62 @@ +import os +import logging + +import ruamel.yaml + + +logger = logging.getLogger(__name__) + + +def load_configuration(filename): + ''' + Load the given configuration file and return its contents as a data structure of nested dicts + and lists. + + Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError + if there are too many recursive includes. + ''' + yaml = ruamel.yaml.YAML(typ='safe') + yaml.Constructor = Include_constructor + + return yaml.load(open(filename)) + + +def include_configuration(loader, filename_node): + ''' + Load the given YAML filename (ignoring the given loader so we can use our own), and return its + contents as a data structure of nested dicts and lists. + ''' + return load_configuration(os.path.expanduser(filename_node.value)) + + +class Include_constructor(ruamel.yaml.SafeConstructor): + ''' + A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including + separate YAML configuration files. Example syntax: `retention: !include common.yaml` + ''' + + def __init__(self, preserve_quotes=None, loader=None): + super(Include_constructor, self).__init__(preserve_quotes, loader) + self.add_constructor('!include', include_configuration) + + def flatten_mapping(self, node): + ''' + Support the special case of shallow merging included configuration into an existing mapping + using the YAML '<<' merge key. Example syntax: + + ``` + retention: + keep_daily: 1 + <<: !include common.yaml + ``` + ''' + representer = ruamel.yaml.representer.SafeRepresenter() + + for index, (key_node, value_node) in enumerate(node.value): + if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include': + included_value = representer.represent_mapping( + tag='tag:yaml.org,2002:map', mapping=self.construct_object(value_node) + ) + node.value[index] = (key_node, included_value) + + super(Include_constructor, self).flatten_mapping(node) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 059636a0..7818a748 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -3,7 +3,9 @@ import logging import pkg_resources import pykwalify.core import pykwalify.errors -from ruamel import yaml +import ruamel.yaml + +from borgmatic.config import load logger = logging.getLogger(__name__) @@ -87,9 +89,9 @@ def parse_configuration(config_filename, schema_filename): logging.getLogger('pykwalify').setLevel(logging.ERROR) try: - config = yaml.safe_load(open(config_filename)) - schema = yaml.safe_load(open(schema_filename)) - except yaml.error.YAMLError as error: + config = load.load_configuration(config_filename) + schema = load.load_configuration(schema_filename) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) # pykwalify gets angry if the example field is not a string. So rather than bend to its will, diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index a92baa43..fbf20b08 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -27,6 +27,85 @@ configuration paths on the command-line with borgmatic's `--config` option. See `borgmatic --help` for more information. +## Configuration includes + +Once you have multiple different configuration files, you might want to share +common configuration options across these files with having to copy and paste +them. To achieve this, you can put fragments of common configuration options +into a file, and then include or inline that file into one or more borgmatic +configuration files. + +Let's say that you want to include common retention configuration across all +of your configuration files. You could do that in each configuration file with +the following: + +```yaml +location: + ... + +retention: + !include /etc/borgmatic/common_retention.yaml +``` + +And then the contents of `common_retention.yaml` could be: + +```yaml +keep_hourly: 24 +keep_daily: 7 +``` + +To prevent borgmatic from trying to load these configuration fragments by +themselves and complaining that they are not valid configuration files, you +should put them in a directory other than `/etc/borgmatic.d/`. (A subdirectory +is fine.) + +Note that this form of include must be a YAML value rather than a key. For +example, this will not work: + +```yaml +location: + ... + +# Don't do this. It won't work! +!include /etc/borgmatic/common_retention.yaml +``` + +But if you do want to merge in a YAML key and its values, keep reading! + + +## Include merging + +If you need to get even fancier and pull in common configuration options while +potentially overriding individual options, you can perform a YAML merge of +included configuration using the YAML `<<` key. For instance, here's an +example of a main configuration file that pulls in two retention options via +an include, and then overrides one of them locally: + +```yaml +location: + ... + +retention: + keep_daily: 5 + <<: !include /etc/borgmatic/common_retention.yaml +``` + +This is what `common_retention.yaml` might look like: + +```yaml +keep_hourly: 24 +keep_daily: 7 +``` + +Once this include gets merged in, the resulting configuration would have a +`keep_hourly` value of `24` and an overridden `keep_daily` value of `5`. + +When there is a collision of an option between the local file and the merged +include, the local file's option takes precedent. And note that this is a +shallow merge rather than a deep merge, so the merging does not descend into +nested values. + + ## Related documentation * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md) diff --git a/setup.py b/setup.py index ceda7a58..620cfd72 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.2.18' +VERSION = '1.3.0' setup( diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py new file mode 100644 index 00000000..bcaae59a --- /dev/null +++ b/tests/integration/config/test_load.py @@ -0,0 +1,40 @@ +import sys + +from flexmock import flexmock + +from borgmatic.config import load as module + + +def test_load_configuration_parses_contents(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('config.yaml').and_return('key: value') + + assert module.load_configuration('config.yaml') == {'key': 'value'} + + +def test_load_configuration_inlines_include(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('include.yaml').and_return('value') + builtins.should_receive('open').with_args('config.yaml').and_return( + 'key: !include include.yaml' + ) + + assert module.load_configuration('config.yaml') == {'key': 'value'} + + +def test_load_configuration_merges_include(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('include.yaml').and_return( + ''' + foo: bar + baz: quux + ''' + ) + builtins.should_receive('open').with_args('config.yaml').and_return( + ''' + foo: override + <<: !include include.yaml + ''' + ) + + assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'} diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index b1ca0988..2182d2ee 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -118,6 +118,67 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): module.parse_configuration('config.yaml', 'schema.yaml') +def test_parse_configuration_inlines_include(): + mock_config_and_schema( + ''' + location: + source_directories: + - /home + + repositories: + - hostname.borg + + retention: + !include include.yaml + ''' + ) + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('include.yaml').and_return( + ''' + keep_daily: 7 + keep_hourly: 24 + ''' + ) + + result = module.parse_configuration('config.yaml', 'schema.yaml') + + assert result == { + 'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']}, + 'retention': {'keep_daily': 7, 'keep_hourly': 24}, + } + + +def test_parse_configuration_merges_include(): + mock_config_and_schema( + ''' + location: + source_directories: + - /home + + repositories: + - hostname.borg + + retention: + keep_daily: 1 + <<: !include include.yaml + ''' + ) + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('include.yaml').and_return( + ''' + keep_daily: 7 + keep_hourly: 24 + ''' + ) + + result = module.parse_configuration('config.yaml', 'schema.yaml') + + assert result == { + 'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']}, + 'retention': {'keep_daily': 1, 'keep_hourly': 24}, + } + + def test_parse_configuration_raises_for_missing_config_file(): with pytest.raises(FileNotFoundError): module.parse_configuration('config.yaml', 'schema.yaml')