Browse Source

Basic YAML configuration file parsing.

Dan Helfman 1 year ago
parent
commit
4d7556f68b

+ 1
- 0
.hgignore View File

@@ -2,6 +2,7 @@ syntax: glob
2 2
 *.egg-info
3 3
 *.pyc
4 4
 *.swp
5
+.cache
5 6
 .tox
6 7
 build
7 8
 dist

+ 1
- 0
MANIFEST.in View File

@@ -0,0 +1 @@
1
+include borgmatic/config/schema.yaml

+ 1
- 1
NEWS View File

@@ -1,4 +1,4 @@
1
-1.0.3-dev
1
+1.1.0
2 2
 
3 3
  * #18: Fix for README mention of sample files not included in package.
4 4
  * #22: Sample files for triggering borgmatic from a systemd timer.

+ 1
- 1
borgmatic/command.py View File

@@ -5,7 +5,7 @@ from subprocess import CalledProcessError
5 5
 import sys
6 6
 
7 7
 from borgmatic import borg
8
-from borgmatic.config import parse_configuration, CONFIG_FORMAT
8
+from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT
9 9
 
10 10
 
11 11
 DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'

+ 0
- 0
borgmatic/config/__init__.py View File


borgmatic/config.py → borgmatic/config/legacy.py View File


+ 48
- 0
borgmatic/config/schema.yaml View File

@@ -0,0 +1,48 @@
1
+map:
2
+    location:
3
+        required: True
4
+        map:
5
+            source_directories:
6
+                required: True
7
+                seq:
8
+                    - type: scalar
9
+            one_file_system:
10
+                type: bool
11
+            remote_path:
12
+                type: scalar
13
+            repository:
14
+                required: True
15
+                type: scalar
16
+    storage:
17
+        map:
18
+            encryption_passphrase:
19
+                type: scalar
20
+            compression:
21
+                type: scalar
22
+            umask:
23
+                type: scalar
24
+    retention:
25
+        map:
26
+            keep_within:
27
+                type: scalar
28
+            keep_hourly:
29
+                type: int
30
+            keep_daily:
31
+                type: int
32
+            keep_weekly:
33
+                type: int
34
+            keep_monthly:
35
+                type: int
36
+            keep_yearly:
37
+                type: int
38
+            prefix:
39
+                type: scalar
40
+    consistency:
41
+        map:
42
+            checks:
43
+                seq:
44
+                    - type: str
45
+                      enum: ['repository', 'archives', 'disabled']
46
+                      unique: True
47
+            check_last:
48
+                type: int

+ 84
- 0
borgmatic/config/yaml.py View File

@@ -0,0 +1,84 @@
1
+import logging
2
+import sys
3
+import warnings
4
+
5
+import pkg_resources
6
+import pykwalify.core
7
+import pykwalify.errors
8
+import ruamel.yaml.error
9
+
10
+
11
+def schema_filename():
12
+    '''
13
+    Path to the installed YAML configuration schema file, used to validate and parse the
14
+    configuration.
15
+    '''
16
+    return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
17
+
18
+
19
+class Validation_error(ValueError):
20
+    '''
21
+    A collection of error message strings generated when attempting to validate a particular
22
+    configurartion file.
23
+    '''
24
+    def __init__(self, config_filename, error_messages):
25
+        self.config_filename = config_filename
26
+        self.error_messages = error_messages
27
+
28
+
29
+def parse_configuration(config_filename, schema_filename):
30
+    '''
31
+    Given the path to a config filename in YAML format and the path to a schema filename in
32
+    pykwalify YAML schema format, return the parsed configuration as a data structure of nested
33
+    dicts and lists corresponding to the schema. Example return value:
34
+
35
+       {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
36
+       'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
37
+
38
+    Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
39
+    have permissions to read the file, or Validation_error if the config does not match the schema.
40
+    '''
41
+    warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
42
+    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
43
+
44
+    try:
45
+        validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
46
+    except pykwalify.errors.CoreError as error:
47
+        if 'do not exists on disk' in str(error):
48
+            raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
49
+        if 'Unable to load any data' in str(error):
50
+            # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
51
+            # So reach back to the originating exception from ruamel.yaml for something more useful.
52
+            raise Validation_error(config_filename, (error.__context__,))
53
+        raise
54
+
55
+    parsed_result = validator.validate(raise_exception=False)
56
+
57
+    if validator.validation_errors:
58
+        raise Validation_error(config_filename, validator.validation_errors)
59
+
60
+    return parsed_result
61
+
62
+
63
+def display_validation_error(validation_error):
64
+    '''
65
+    Given a Validation_error, display its error messages to stderr.
66
+    '''
67
+    print(
68
+        'An error occurred while parsing a configuration file at {}:'.format(
69
+            validation_error.config_filename
70
+        ),
71
+        file=sys.stderr,
72
+    )
73
+
74
+    for error in validation_error.error_messages:
75
+        print(error, file=sys.stderr)
76
+
77
+
78
+# FOR TESTING
79
+if __name__ == '__main__':
80
+    try:
81
+        configuration = parse_configuration('sample/config.yaml', schema_filename())
82
+        print(configuration)
83
+    except Validation_error as error:
84
+        display_validation_error(error)

+ 0
- 6
borgmatic/tests/builtins.py View File

@@ -1,6 +0,0 @@
1
-from flexmock import flexmock
2
-import sys
3
-
4
-
5
-def builtins_mock():
6
-    return flexmock(sys.modules['builtins'])

+ 0
- 0
borgmatic/tests/integration/config/__init__.py View File


borgmatic/tests/integration/test_config.py → borgmatic/tests/integration/config/test_legacy.py View File

@@ -3,7 +3,7 @@ from io import StringIO
3 3
 from collections import OrderedDict
4 4
 import string
5 5
 
6
-from borgmatic import config as module
6
+from borgmatic.config import legacy as module
7 7
 
8 8
 
9 9
 def test_parse_section_options_with_punctuation_should_return_section_options():

+ 0
- 0
borgmatic/tests/unit/config/__init__.py View File


borgmatic/tests/unit/test_config.py → borgmatic/tests/unit/config/test_legacy.py View File

@@ -3,7 +3,7 @@ from collections import OrderedDict
3 3
 from flexmock import flexmock
4 4
 import pytest
5 5
 
6
-from borgmatic import config as module
6
+from borgmatic.config import legacy as module
7 7
 
8 8
 
9 9
 def test_option_should_create_config_option():

+ 3
- 3
borgmatic/tests/unit/test_borg.py View File

@@ -1,11 +1,11 @@
1 1
 from collections import OrderedDict
2 2
 from subprocess import STDOUT
3
+import sys
3 4
 import os
4 5
 
5 6
 from flexmock import flexmock
6 7
 
7 8
 from borgmatic import borg as module
8
-from borgmatic.tests.builtins import builtins_mock
9 9
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
10 10
 
11 11
 
@@ -389,7 +389,7 @@ def test_check_archives_should_call_borg_with_parameters():
389 389
     )
390 390
     insert_platform_mock()
391 391
     insert_datetime_mock()
392
-    builtins_mock().should_receive('open').and_return(stdout)
392
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
393 393
     flexmock(module.os).should_receive('devnull')
394 394
 
395 395
     module.check_archives(
@@ -464,7 +464,7 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
464 464
     )
465 465
     insert_platform_mock()
466 466
     insert_datetime_mock()
467
-    builtins_mock().should_receive('open').and_return(stdout)
467
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
468 468
     flexmock(module.os).should_receive('devnull')
469 469
 
470 470
     module.check_archives(

+ 54
- 0
sample/config.yaml View File

@@ -0,0 +1,54 @@
1
+location:
2
+    # List of source directories to backup. Globs are expanded.
3
+    source_directories:
4
+        - /home
5
+        - /etc
6
+        - /var/log/syslog*
7
+
8
+    # Stay in same file system (do not cross mount points).
9
+    #one_file_system: yes
10
+
11
+    # Alternate Borg remote executable (defaults to "borg"):
12
+    #remote_path: borg1
13
+
14
+    # Path to local or remote repository.
15
+    repository: user@backupserver:sourcehostname.borg
16
+
17
+#storage:
18
+    # Passphrase to unlock the encryption key with. Only use on repositories
19
+    # that were initialized with passphrase/repokey encryption.
20
+    #encryption_passphrase: foo
21
+
22
+    # Type of compression to use when creating archives. See
23
+    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
24
+    # for details. Defaults to no compression.
25
+    #compression: lz4
26
+
27
+    # Umask to be used for borg create.
28
+    #umask: 0077
29
+
30
+retention:
31
+    # Retention policy for how many backups to keep in each category. See
32
+    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for
33
+    # details.
34
+    #keep_within: 3H
35
+    #keep_hourly: 24
36
+    keep_daily: 7
37
+    keep_weekly: 4
38
+    keep_monthly: 6
39
+    keep_yearly: 1
40
+
41
+    # When pruning, only consider archive names starting with this prefix.
42
+    #prefix: sourcehostname
43
+
44
+consistency:
45
+    # List of consistency checks to run: "repository", "archives", or both.
46
+    # Defaults to both. Set to "disabled" to disable all consistency checks.
47
+    # See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
48
+    # for details.
49
+    checks:
50
+        - repository
51
+        - archives
52
+
53
+    # Restrict the number of checked archives to the last n.
54
+    #check_last: 3

+ 8
- 2
setup.py View File

@@ -1,7 +1,7 @@
1 1
 from setuptools import setup, find_packages
2 2
 
3 3
 
4
-VERSION = '1.0.3-dev'
4
+VERSION = '1.1.0'
5 5
 
6 6
 
7 7
 setup(
@@ -30,8 +30,14 @@ setup(
30 30
     obsoletes=[
31 31
         'atticmatic',
32 32
     ],
33
+    install_requires=(
34
+        'pykwalify',
35
+        'ruamel.yaml<=0.15',
36
+        'setuptools',
37
+    ),
33 38
     tests_require=(
34 39
         'flexmock',
35 40
         'pytest',
36
-    )
41
+    ),
42
+    include_package_data=True,
37 43
 )

Loading…
Cancel
Save