Configuration files includes and merging (#148).

NEWS

@@ -1,3 +1,7 @@
+1.3.0
+ * #148: Configuration file includes and merging via "!include" tag to support reuse of common
+   options across configuration files.
+
 1.2.18
  * #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
    borgmatic's location section.

borgmatic/config/load.py

@@ -0,0 +1,62 @@
+import os
+import logging
+
+import ruamel.yaml
+
+
+logger = logging.getLogger(__name__)
+
+
+def load_configuration(filename):
+    '''
+    Load the given configuration file and return its contents as a data structure of nested dicts
+    and lists.
+
+    Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
+    if there are too many recursive includes.
+    '''
+    yaml = ruamel.yaml.YAML(typ='safe')
+    yaml.Constructor = Include_constructor
+
+    return yaml.load(open(filename))
+
+
+def include_configuration(loader, filename_node):
+    '''
+    Load the given YAML filename (ignoring the given loader so we can use our own), and return its
+    contents as a data structure of nested dicts and lists.
+    '''
+    return load_configuration(os.path.expanduser(filename_node.value))
+
+
+class Include_constructor(ruamel.yaml.SafeConstructor):
+    '''
+    A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
+    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    '''
+
+    def __init__(self, preserve_quotes=None, loader=None):
+        super(Include_constructor, self).__init__(preserve_quotes, loader)
+        self.add_constructor('!include', include_configuration)
+
+    def flatten_mapping(self, node):
+        '''
+        Support the special case of shallow merging included configuration into an existing mapping
+        using the YAML '<<' merge key. Example syntax:
+
+        ```
+        retention:
+            keep_daily: 1
+            <<: !include common.yaml
+        ```
+        '''
+        representer = ruamel.yaml.representer.SafeRepresenter()
+
+        for index, (key_node, value_node) in enumerate(node.value):
+            if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
+                included_value = representer.represent_mapping(
+                    tag='tag:yaml.org,2002:map', mapping=self.construct_object(value_node)
+                )
+                node.value[index] = (key_node, included_value)
+
+        super(Include_constructor, self).flatten_mapping(node)

borgmatic/config/validate.py

@@ -3,7 +3,9 @@ import logging
 import pkg_resources
 import pykwalify.core
 import pykwalify.errors
-from ruamel import yaml
+import ruamel.yaml
+
+from borgmatic.config import load
 
 
 logger = logging.getLogger(__name__)
@@ -87,9 +89,9 @@ def parse_configuration(config_filename, schema_filename):
     logging.getLogger('pykwalify').setLevel(logging.ERROR)
 
     try:
-        config = yaml.safe_load(open(config_filename))
-        schema = yaml.safe_load(open(schema_filename))
-    except yaml.error.YAMLError as error:
+        config = load.load_configuration(config_filename)
+        schema = load.load_configuration(schema_filename)
+    except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
     # pykwalify gets angry if the example field is not a string. So rather than bend to its will,

docs/how-to/make-per-application-backups.md

@@ -27,6 +27,85 @@ configuration paths on the command-line with borgmatic's `--config` option.
 See `borgmatic --help` for more information.
 
 
+## Configuration includes
+
+Once you have multiple different configuration files, you might want to share
+common configuration options across these files with having to copy and paste
+them. To achieve this, you can put fragments of common configuration options
+into a file, and then include or inline that file into one or more borgmatic
+configuration files.
+
+Let's say that you want to include common retention configuration across all
+of your configuration files. You could do that in each configuration file with
+the following:
+
+```yaml
+location:
+   ...
+
+retention:
+    !include /etc/borgmatic/common_retention.yaml
+```
+
+And then the contents of `common_retention.yaml` could be:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+To prevent borgmatic from trying to load these configuration fragments by
+themselves and complaining that they are not valid configuration files, you
+should put them in a directory other than `/etc/borgmatic.d/`. (A subdirectory
+is fine.)
+
+Note that this form of include must be a YAML value rather than a key. For
+example, this will not work:
+
+```yaml
+location:
+   ...
+
+# Don't do this. It won't work!
+!include /etc/borgmatic/common_retention.yaml
+```
+
+But if you do want to merge in a YAML key and its values, keep reading!
+
+
+## Include merging
+
+If you need to get even fancier and pull in common configuration options while
+potentially overriding individual options, you can perform a YAML merge of
+included configuration using the YAML `<<` key. For instance, here's an
+example of a main configuration file that pulls in two retention options via
+an include, and then overrides one of them locally:
+
+```yaml
+location:
+   ...
+
+retention:
+    keep_daily: 5
+    <<: !include /etc/borgmatic/common_retention.yaml
+```
+
+This is what `common_retention.yaml` might look like:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+Once this include gets merged in, the resulting configuration would have a
+`keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
+
+When there is a collision of an option between the local file and the merged
+include, the local file's option takes precedent. And note that this is a
+shallow merge rather than a deep merge, so the merging does not descend into
+nested values.
+
+
 ## Related documentation
 
  * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.2.18'
+VERSION = '1.3.0'
 
 
 setup(

tests/integration/config/test_load.py

@@ -0,0 +1,40 @@
+import sys
+
+from flexmock import flexmock
+
+from borgmatic.config import load as module
+
+
+def test_load_configuration_parses_contents():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('config.yaml').and_return('key: value')
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_inlines_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return('value')
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        'key: !include include.yaml'
+    )
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_merges_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        foo: bar
+        baz: quux
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: override
+        <<: !include include.yaml
+        '''
+    )
+
+    assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}

tests/integration/config/test_validate.py

@@ -118,6 +118,67 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
     module.parse_configuration('config.yaml', 'schema.yaml')
 
 
+def test_parse_configuration_inlines_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 7, 'keep_hourly': 24},
+    }
+
+
+def test_parse_configuration_merges_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            keep_daily: 1
+            <<: !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 1, 'keep_hourly': 24},
+    }
+
+
 def test_parse_configuration_raises_for_missing_config_file():
     with pytest.raises(FileNotFoundError):
         module.parse_configuration('config.yaml', 'schema.yaml')