Browse Source

Basic YAML generating / validating / converting to.

Dan Helfman 1 year ago
parent
commit
f19a40ef9c

+ 3
- 2
NEWS View File

@@ -1,8 +1,9 @@
1
-1.1.0
1
+1.1.0.dev0
2 2
 
3
+ * Switched config file format to YAML. Run convert-borgmatic-config to upgrade.
4
+ * Dropped Python 2 support. Now Python 3 only.
3 5
  * #18: Fix for README mention of sample files not included in package.
4 6
  * #22: Sample files for triggering borgmatic from a systemd timer.
5
- * Dropped Python 2 support. Now Python 3 only.
6 7
  * Added logo.
7 8
 
8 9
 1.0.3

+ 0
- 0
borgmatic/commands/__init__.py View File


borgmatic/command.py → borgmatic/commands/borgmatic.py View File

@@ -5,7 +5,7 @@ from subprocess import CalledProcessError
5 5
 import sys
6 6
 
7 7
 from borgmatic import borg
8
-from borgmatic.config.yaml import parse_configuration, schema_filename
8
+from borgmatic.config.validate import parse_configuration, schema_filename
9 9
 
10 10
 
11 11
 DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
@@ -14,9 +14,8 @@ DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
14 14
 
15 15
 def parse_arguments(*arguments):
16 16
     '''
17
-    Given the name of the command with which this script was invoked and command-line arguments,
18
-    parse the arguments and return them as an ArgumentParser instance. Use the command name to
19
-    determine the default configuration and excludes paths.
17
+    Given command-line arguments with which this script was invoked, parse the arguments and return
18
+    them as an ArgumentParser instance.
20 19
     '''
21 20
     parser = ArgumentParser()
22 21
     parser.add_argument(

+ 54
- 0
borgmatic/commands/convert_config.py View File

@@ -0,0 +1,54 @@
1
+from __future__ import print_function
2
+from argparse import ArgumentParser
3
+import os
4
+from subprocess import CalledProcessError
5
+import sys
6
+
7
+from ruamel import yaml
8
+
9
+from borgmatic import borg
10
+from borgmatic.config import convert, generate, legacy, validate
11
+
12
+
13
+DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
14
+DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
15
+DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
16
+
17
+
18
+def parse_arguments(*arguments):
19
+    '''
20
+    Given command-line arguments with which this script was invoked, parse the arguments and return
21
+    them as an ArgumentParser instance.
22
+    '''
23
+    parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.')
24
+    parser.add_argument(
25
+        '-s', '--source',
26
+        dest='source_filename',
27
+        default=DEFAULT_SOURCE_CONFIG_FILENAME,
28
+        help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
29
+    )
30
+    parser.add_argument(
31
+        '-d', '--destination',
32
+        dest='destination_filename',
33
+        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
34
+        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
35
+    )
36
+
37
+    return parser.parse_args(arguments)
38
+
39
+
40
+def main():
41
+    try:
42
+        args = parse_arguments(*sys.argv[1:])
43
+        source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
44
+        schema = yaml.round_trip_load(open(validate.schema_filename()).read())
45
+
46
+        destination_config = convert.convert_legacy_parsed_config(source_config, schema)
47
+
48
+        generate.write_configuration(args.destination_filename, destination_config)
49
+
50
+        # TODO: As a backstop, check that the written config can actually be read and parsed, and
51
+        # that it matches the destination config data structure that was written.
52
+    except (ValueError, OSError) as error:
53
+        print(error, file=sys.stderr)
54
+        sys.exit(1)

+ 41
- 0
borgmatic/config/convert.py View File

@@ -0,0 +1,41 @@
1
+from ruamel import yaml
2
+
3
+from borgmatic.config import generate
4
+
5
+
6
+def _convert_section(source_section_config, section_schema):
7
+    '''
8
+    Given a legacy Parsed_config instance for a single section, convert it to its corresponding
9
+    yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.
10
+
11
+    Additionally, use the section schema as a source of helpful comments to include within the
12
+    returned CommentedMap.
13
+    '''
14
+    destination_section_config = yaml.comments.CommentedMap(source_section_config)
15
+    generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT)
16
+
17
+    return destination_section_config
18
+
19
+
20
+def convert_legacy_parsed_config(source_config, schema):
21
+    '''
22
+    Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its
23
+    corresponding yaml.comments.CommentedMap representation in preparation for actual serialization
24
+    to YAML.
25
+
26
+    Additionally, use the given schema as a source of helpful comments to include within the
27
+    returned CommentedMap.
28
+    '''
29
+    destination_config = yaml.comments.CommentedMap([
30
+        (section_name, _convert_section(section_config, schema['map'][section_name]))
31
+        for section_name, section_config in source_config._asdict().items()
32
+    ])
33
+
34
+    destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
35
+
36
+    if source_config.consistency['checks']:
37
+        destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
38
+
39
+    generate.add_comments_to_configuration(destination_config, schema)
40
+
41
+    return destination_config

+ 90
- 0
borgmatic/config/generate.py View File

@@ -0,0 +1,90 @@
1
+from collections import OrderedDict
2
+
3
+from ruamel import yaml
4
+
5
+
6
+INDENT = 4
7
+
8
+
9
+def write_configuration(config_filename, config):
10
+    '''
11
+    Given a target config filename and a config data structure of nested OrderedDicts, write out the
12
+    config to file as YAML.
13
+    '''
14
+    with open(config_filename, 'w') as config_file:
15
+        config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
16
+
17
+
18
+def _insert_newline_before_comment(config, field_name):
19
+    '''
20
+    Using some ruamel.yaml black magic, insert a blank line in the config right befor the given
21
+    field and its comments.
22
+    '''
23
+    config.ca.items[field_name][1].insert(
24
+        0,
25
+        yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None),
26
+    )
27
+
28
+
29
+def add_comments_to_configuration(config, schema, indent=0):
30
+    '''
31
+    Using descriptions from a schema as a source, add those descriptions as comments to the given
32
+    config before each field. This function only adds comments for the top-most config map level.
33
+    Indent the comment the given number of characters.
34
+    '''
35
+    for index, field_name in enumerate(config.keys()):
36
+        field_schema = schema['map'].get(field_name, {})
37
+        description = field_schema.get('desc')
38
+
39
+        # No description to use? Skip it.
40
+        if not schema or not description:
41
+            continue
42
+
43
+        config.yaml_set_comment_before_after_key(
44
+            key=field_name,
45
+            before=description,
46
+            indent=indent,
47
+        )
48
+        if index > 0:
49
+            _insert_newline_before_comment(config, field_name)
50
+
51
+
52
+def _section_schema_to_sample_configuration(section_schema):
53
+    '''
54
+    Given the schema for a particular config section, generate and return sample config for that
55
+    section. Include comments for each field based on the schema "desc" description.
56
+    '''
57
+    section_config = yaml.comments.CommentedMap([
58
+        (field_name, field_schema['example'])
59
+        for field_name, field_schema in section_schema['map'].items()
60
+    ])
61
+
62
+    add_comments_to_configuration(section_config, section_schema, indent=INDENT)
63
+
64
+    return section_config
65
+
66
+
67
+def _schema_to_sample_configuration(schema):
68
+    '''
69
+    Given a loaded configuration schema, generate and return sample config for it. Include comments
70
+    for each section based on the schema "desc" description.
71
+    '''
72
+    config = yaml.comments.CommentedMap([
73
+        (section_name, _section_schema_to_sample_configuration(section_schema))
74
+        for section_name, section_schema in schema['map'].items()
75
+    ])
76
+
77
+    add_comments_to_configuration(config, schema)
78
+
79
+    return config
80
+
81
+
82
+def generate_sample_configuration(config_filename, schema_filename):
83
+    '''
84
+    Given a target config filename and the path to a schema filename in pykwalify YAML schema
85
+    format, write out a sample configuration file based on that schema.
86
+    '''
87
+    schema = yaml.round_trip_load(open(schema_filename))
88
+    config = _schema_to_sample_configuration(schema)
89
+
90
+    write_configuration(config_filename, config)

+ 62
- 0
borgmatic/config/schema.yaml View File

@@ -1,48 +1,110 @@
1
+name: Borgmatic configuration file schema
1 2
 map:
2 3
     location:
4
+        desc: |
5
+            Where to look for files to backup, and where to store those backups. See
6
+            https://borgbackup.readthedocs.io/en/stable/quickstart.html and
7
+            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details.
3 8
         required: True
4 9
         map:
5 10
             source_directories:
6 11
                 required: True
7 12
                 seq:
8 13
                     - type: scalar
14
+                desc: List of source directories to backup. Globs are expanded.
15
+                example:
16
+                    - /home
17
+                    - /etc
18
+                    - /var/log/syslog*
9 19
             one_file_system:
10 20
                 type: bool
21
+                desc: Stay in same file system (do not cross mount points).
22
+                example: yes
11 23
             remote_path:
12 24
                 type: scalar
25
+                desc: Alternate Borg remote executable. Defaults to "borg".
26
+                example: borg1
13 27
             repository:
14 28
                 required: True
15 29
                 type: scalar
30
+                desc: Path to local or remote repository.
31
+                example: user@backupserver:sourcehostname.borg
16 32
     storage:
33
+        desc: |
34
+            Repository storage options. See
35
+            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
36
+            https://borgbackup.readthedocs.io/en/stable/usage.html#environment-variables for details.
17 37
         map:
18 38
             encryption_passphrase:
19 39
                 type: scalar
40
+                desc: |
41
+                    Passphrase to unlock the encryption key with. Only use on repositories that were
42
+                    initialized with passphrase/repokey encryption. Quote the value if it contains
43
+                    punctuation, so it parses correctly. And backslash any quote or backslash
44
+                    literals as well.
45
+                example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
20 46
             compression:
21 47
                 type: scalar
48
+                desc: |
49
+                    Type of compression to use when creating archives. See
50
+                    https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
51
+                    Defaults to no compression.
52
+                example: lz4
22 53
             umask:
23 54
                 type: scalar
55
+                desc: Umask to be used for borg create.
56
+                example: 0077
24 57
     retention:
58
+        desc: |
59
+            Retention policy for how many backups to keep in each category. See
60
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
25 61
         map:
26 62
             keep_within:
27 63
                 type: scalar
64
+                desc: Keep all archives within this time interval.
65
+                example: 3H
28 66
             keep_hourly:
29 67
                 type: int
68
+                desc: Number of hourly archives to keep.
69
+                example: 24
30 70
             keep_daily:
31 71
                 type: int
72
+                desc: Number of daily archives to keep.
73
+                example: 7
32 74
             keep_weekly:
33 75
                 type: int
76
+                desc: Number of weekly archives to keep.
77
+                example: 4
34 78
             keep_monthly:
35 79
                 type: int
80
+                desc: Number of monthly archives to keep.
81
+                example: 6
36 82
             keep_yearly:
37 83
                 type: int
84
+                desc: Number of yearly archives to keep.
85
+                example: 1
38 86
             prefix:
39 87
                 type: scalar
88
+                desc: When pruning, only consider archive names starting with this prefix.
89
+                example: sourcehostname
40 90
     consistency:
91
+        desc: |
92
+            Consistency checks to run after backups. See
93
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
41 94
         map:
42 95
             checks:
43 96
                 seq:
44 97
                     - type: str
45 98
                       enum: ['repository', 'archives', 'disabled']
46 99
                       unique: True
100
+                desc: |
101
+                    List of consistency checks to run: "repository", "archives", or both. Defaults
102
+                    to both. Set to "disabled" to disable all consistency checks. See
103
+                    https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
104
+                example:
105
+                    - repository
106
+                    - archives
47 107
             check_last:
48 108
                 type: int
109
+                desc: Restrict the number of checked archives to the last n.
110
+                example: 3

borgmatic/config/yaml.py → borgmatic/config/validate.py View File

@@ -5,7 +5,7 @@ import warnings
5 5
 import pkg_resources
6 6
 import pykwalify.core
7 7
 import pykwalify.errors
8
-import ruamel.yaml.error
8
+from ruamel import yaml
9 9
 
10 10
 
11 11
 def schema_filename():
@@ -38,20 +38,18 @@ def parse_configuration(config_filename, schema_filename):
38 38
     Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
39 39
     have permissions to read the file, or Validation_error if the config does not match the schema.
40 40
     '''
41
-    warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
42
-    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
43
-
44 41
     try:
45
-        validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
46
-    except pykwalify.errors.CoreError as error:
47
-        if 'do not exists on disk' in str(error):
48
-            raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
49
-        if 'Unable to load any data' in str(error):
50
-            # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
51
-            # So reach back to the originating exception from ruamel.yaml for something more useful.
52
-            raise Validation_error(config_filename, (error.__context__,))
53
-        raise
42
+        schema = yaml.round_trip_load(open(schema_filename))
43
+    except yaml.error.YAMLError as error:
44
+        raise Validation_error(config_filename, (str(error),))
45
+
46
+    # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
47
+    # simply remove all examples before passing the schema to pykwalify.
48
+    for section_name, section_schema in schema['map'].items():
49
+        for field_name, field_schema in section_schema['map'].items():
50
+            field_schema.pop('example')
54 51
 
52
+    validator = pykwalify.core.Core(source_file=config_filename, schema_data=schema)
55 53
     parsed_result = validator.validate(raise_exception=False)
56 54
 
57 55
     if validator.validation_errors:
@@ -73,12 +71,3 @@ def display_validation_error(validation_error):
73 71
 
74 72
     for error in validation_error.error_messages:
75 73
         print(error, file=sys.stderr)
76
-
77
-
78
-# FOR TESTING
79
-if __name__ == '__main__':
80
-    try:
81
-        configuration = parse_configuration('sample/config.yaml', schema_filename())
82
-        print(configuration)
83
-    except Validation_error as error:
84
-        display_validation_error(error)

+ 0
- 0
borgmatic/tests/integration/commands/__init__.py View File


borgmatic/tests/integration/test_command.py → borgmatic/tests/integration/commands/test_borgmatic.py View File

@@ -4,7 +4,7 @@ import sys
4 4
 from flexmock import flexmock
5 5
 import pytest
6 6
 
7
-from borgmatic import command as module
7
+from borgmatic.commands import borgmatic as module
8 8
 
9 9
 
10 10
 def test_parse_arguments_with_no_arguments_uses_defaults():

borgmatic/tests/integration/config/test_yaml.py → borgmatic/tests/integration/config/test_validate.py View File

@@ -6,7 +6,7 @@ import os
6 6
 from flexmock import flexmock
7 7
 import pytest
8 8
 
9
-from borgmatic.config import yaml as module
9
+from borgmatic.config import validate as module
10 10
 
11 11
 
12 12
 def test_schema_filename_returns_plausable_path():
@@ -18,13 +18,13 @@ def test_schema_filename_returns_plausable_path():
18 18
 def mock_config_and_schema(config_yaml):
19 19
     '''
20 20
     Set up mocks for the config config YAML string and the default schema so that pykwalify consumes
21
-    them when parsing the configuration. This is a little brittle in that it's relying on pykwalify
22
-    to open() the respective files in a particular order.
21
+    them when parsing the configuration. This is a little brittle in that it's relying on the code
22
+    under test to open() the respective files in a particular order.
23 23
     '''
24
-    config_stream = io.StringIO(config_yaml)
25 24
     schema_stream = open(module.schema_filename())
25
+    config_stream = io.StringIO(config_yaml)
26 26
     builtins = flexmock(sys.modules['builtins']).should_call('open').mock
27
-    builtins.should_receive('open').and_return(config_stream).and_return(schema_stream)
27
+    builtins.should_receive('open').and_return(schema_stream).and_return(config_stream)
28 28
     flexmock(os.path).should_receive('exists').and_return(True)
29 29
 
30 30
 
@@ -87,7 +87,8 @@ def test_parse_configuration_raises_for_missing_config_file():
87 87
 
88 88
 def test_parse_configuration_raises_for_missing_schema_file():
89 89
     mock_config_and_schema('')
90
-    flexmock(os.path).should_receive('exists').with_args('schema.yaml').and_return(False)
90
+    builtins = flexmock(sys.modules['builtins'])
91
+    builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError)
91 92
 
92 93
     with pytest.raises(FileNotFoundError):
93 94
         module.parse_configuration('config.yaml', 'schema.yaml')

+ 44
- 0
borgmatic/tests/unit/config/test_convert.py View File

@@ -0,0 +1,44 @@
1
+from collections import defaultdict, OrderedDict, namedtuple
2
+
3
+from borgmatic.config import convert as module
4
+
5
+
6
+Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))
7
+
8
+
9
+def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
10
+    source_config = Parsed_config(
11
+        location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
12
+        storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
13
+        retention=OrderedDict([('keep_daily', 7)]),
14
+        consistency=OrderedDict([('checks', 'repository')]),
15
+    )
16
+    schema = {'map': defaultdict(lambda: {'map': {}})}
17
+
18
+    destination_config = module.convert_legacy_parsed_config(source_config, schema)
19
+
20
+    assert destination_config == OrderedDict([
21
+        ('location', OrderedDict([('source_directories', ['/home']), ('repository', 'hostname.borg')])),
22
+        ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])),
23
+        ('retention', OrderedDict([('keep_daily', 7)])),
24
+        ('consistency', OrderedDict([('checks', ['repository'])])),
25
+    ])
26
+
27
+
28
+def test_convert_legacy_parsed_config_splits_space_separated_values():
29
+    source_config = Parsed_config(
30
+        location=OrderedDict([('source_directories', '/home /etc')]),
31
+        storage=OrderedDict(),
32
+        retention=OrderedDict(),
33
+        consistency=OrderedDict([('checks', 'repository archives')]),
34
+    )
35
+    schema = {'map': defaultdict(lambda: {'map': {}})}
36
+
37
+    destination_config = module.convert_legacy_parsed_config(source_config, schema) 
38
+
39
+    assert destination_config == OrderedDict([
40
+        ('location', OrderedDict([('source_directories', ['/home', '/etc'])])),
41
+        ('storage', OrderedDict()),
42
+        ('retention', OrderedDict()),
43
+        ('consistency', OrderedDict([('checks', ['repository', 'archives'])])),
44
+    ])

+ 4
- 2
sample/config.yaml View File

@@ -16,8 +16,10 @@ location:
16 16
 
17 17
 #storage:
18 18
     # Passphrase to unlock the encryption key with. Only use on repositories
19
-    # that were initialized with passphrase/repokey encryption.
20
-    #encryption_passphrase: foo
19
+    # that were initialized with passphrase/repokey encryption. Quote the value
20
+    # if it contains punctuation so it parses correctly. And backslash any
21
+    # quote or backslash literals as well.
22
+    #encryption_passphrase: "foo"
21 23
 
22 24
     # Type of compression to use when creating archives. See
23 25
     # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create

+ 3
- 2
setup.py View File

@@ -1,7 +1,7 @@
1 1
 from setuptools import setup, find_packages
2 2
 
3 3
 
4
-VERSION = '1.1.0'
4
+VERSION = '1.1.0.dev0'
5 5
 
6 6
 
7 7
 setup(
@@ -24,7 +24,8 @@ setup(
24 24
     packages=find_packages(),
25 25
     entry_points={
26 26
         'console_scripts': [
27
-            'borgmatic = borgmatic.command:main',
27
+            'borgmatic = borgmatic.commands.borgmatic:main',
28
+            'convert-borgmatic-config = borgmatic.commands.convert_config:main',
28 29
         ]
29 30
     },
30 31
     obsoletes=[

+ 3
- 0
test_requirements.txt View File

@@ -1,2 +1,5 @@
1 1
 flexmock==0.10.2
2
+pykwalify==1.6.0
2 3
 pytest==2.9.1
4
+pytest-cov==2.5.1
5
+ruamel.yaml==0.15.18

+ 1
- 1
tox.ini View File

@@ -5,4 +5,4 @@ skipsdist=True
5 5
 [testenv]
6 6
 usedevelop=True
7 7
 deps=-rtest_requirements.txt
8
-commands = py.test borgmatic []
8
+commands = py.test --cov=borgmatic borgmatic []

Loading…
Cancel
Save