From 4d7556f68bd5d75570cd3a989f4d494322a80925 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 4 Jul 2017 16:52:24 -0700 Subject: [PATCH] Basic YAML configuration file parsing. --- .hgignore | 1 + MANIFEST.in | 1 + NEWS | 2 +- borgmatic/command.py | 2 +- borgmatic/config/__init__.py | 0 borgmatic/{config.py => config/legacy.py} | 0 borgmatic/config/schema.yaml | 48 +++++++++++ borgmatic/config/yaml.py | 84 +++++++++++++++++++ borgmatic/tests/builtins.py | 6 -- .../tests/integration/config/__init__.py | 0 .../{test_config.py => config/test_legacy.py} | 2 +- borgmatic/tests/unit/config/__init__.py | 0 .../{test_config.py => config/test_legacy.py} | 2 +- borgmatic/tests/unit/test_borg.py | 6 +- sample/config.yaml | 54 ++++++++++++ setup.py | 10 ++- 16 files changed, 203 insertions(+), 15 deletions(-) create mode 100644 MANIFEST.in create mode 100644 borgmatic/config/__init__.py rename borgmatic/{config.py => config/legacy.py} (100%) create mode 100644 borgmatic/config/schema.yaml create mode 100644 borgmatic/config/yaml.py delete mode 100644 borgmatic/tests/builtins.py create mode 100644 borgmatic/tests/integration/config/__init__.py rename borgmatic/tests/integration/{test_config.py => config/test_legacy.py} (92%) create mode 100644 borgmatic/tests/unit/config/__init__.py rename borgmatic/tests/unit/{test_config.py => config/test_legacy.py} (99%) create mode 100644 sample/config.yaml diff --git a/.hgignore b/.hgignore index 41280764d..f41ba9356 100644 --- a/.hgignore +++ b/.hgignore @@ -2,6 +2,7 @@ syntax: glob *.egg-info *.pyc *.swp +.cache .tox build dist diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..20ff23126 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include borgmatic/config/schema.yaml diff --git a/NEWS b/NEWS index 912436059..b5d421636 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.0.3-dev +1.1.0 * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. diff --git a/borgmatic/command.py b/borgmatic/command.py index 784b74a42..ef7bbee6f 100644 --- a/borgmatic/command.py +++ b/borgmatic/command.py @@ -5,7 +5,7 @@ from subprocess import CalledProcessError import sys from borgmatic import borg -from borgmatic.config import parse_configuration, CONFIG_FORMAT +from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config' diff --git a/borgmatic/config/__init__.py b/borgmatic/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/borgmatic/config.py b/borgmatic/config/legacy.py similarity index 100% rename from borgmatic/config.py rename to borgmatic/config/legacy.py diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml new file mode 100644 index 000000000..83c50bb19 --- /dev/null +++ b/borgmatic/config/schema.yaml @@ -0,0 +1,48 @@ +map: + location: + required: True + map: + source_directories: + required: True + seq: + - type: scalar + one_file_system: + type: bool + remote_path: + type: scalar + repository: + required: True + type: scalar + storage: + map: + encryption_passphrase: + type: scalar + compression: + type: scalar + umask: + type: scalar + retention: + map: + keep_within: + type: scalar + keep_hourly: + type: int + keep_daily: + type: int + keep_weekly: + type: int + keep_monthly: + type: int + keep_yearly: + type: int + prefix: + type: scalar + consistency: + map: + checks: + seq: + - type: str + enum: ['repository', 'archives', 'disabled'] + unique: True + check_last: + type: int diff --git a/borgmatic/config/yaml.py b/borgmatic/config/yaml.py new file mode 100644 index 000000000..4ff01b98d --- /dev/null +++ b/borgmatic/config/yaml.py @@ -0,0 +1,84 @@ +import logging +import sys +import warnings + +import pkg_resources +import pykwalify.core +import pykwalify.errors +import ruamel.yaml.error + + +def schema_filename(): + ''' + Path to the installed YAML configuration schema file, used to validate and parse the + configuration. + ''' + return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml') + + +class Validation_error(ValueError): + ''' + A collection of error message strings generated when attempting to validate a particular + configurartion file. + ''' + def __init__(self, config_filename, error_messages): + self.config_filename = config_filename + self.error_messages = error_messages + + +def parse_configuration(config_filename, schema_filename): + ''' + Given the path to a config filename in YAML format and the path to a schema filename in + pykwalify YAML schema format, return the parsed configuration as a data structure of nested + dicts and lists corresponding to the schema. Example return value: + + {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, + 'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}} + + Raise FileNotFoundError if the file does not exist, PermissionError if the user does not + have permissions to read the file, or Validation_error if the config does not match the schema. + ''' + warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning) + logging.getLogger('pykwalify').setLevel(logging.CRITICAL) + + try: + validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename]) + except pykwalify.errors.CoreError as error: + if 'do not exists on disk' in str(error): + raise FileNotFoundError("No such file or directory: '{}'".format(config_filename)) + if 'Unable to load any data' in str(error): + # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful. + # So reach back to the originating exception from ruamel.yaml for something more useful. + raise Validation_error(config_filename, (error.__context__,)) + raise + + parsed_result = validator.validate(raise_exception=False) + + if validator.validation_errors: + raise Validation_error(config_filename, validator.validation_errors) + + return parsed_result + + +def display_validation_error(validation_error): + ''' + Given a Validation_error, display its error messages to stderr. + ''' + print( + 'An error occurred while parsing a configuration file at {}:'.format( + validation_error.config_filename + ), + file=sys.stderr, + ) + + for error in validation_error.error_messages: + print(error, file=sys.stderr) + + +# FOR TESTING +if __name__ == '__main__': + try: + configuration = parse_configuration('sample/config.yaml', schema_filename()) + print(configuration) + except Validation_error as error: + display_validation_error(error) diff --git a/borgmatic/tests/builtins.py b/borgmatic/tests/builtins.py deleted file mode 100644 index 53970cdee..000000000 --- a/borgmatic/tests/builtins.py +++ /dev/null @@ -1,6 +0,0 @@ -from flexmock import flexmock -import sys - - -def builtins_mock(): - return flexmock(sys.modules['builtins']) diff --git a/borgmatic/tests/integration/config/__init__.py b/borgmatic/tests/integration/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/borgmatic/tests/integration/test_config.py b/borgmatic/tests/integration/config/test_legacy.py similarity index 92% rename from borgmatic/tests/integration/test_config.py rename to borgmatic/tests/integration/config/test_legacy.py index 56cad6c34..13ee65465 100644 --- a/borgmatic/tests/integration/test_config.py +++ b/borgmatic/tests/integration/config/test_legacy.py @@ -3,7 +3,7 @@ from io import StringIO from collections import OrderedDict import string -from borgmatic import config as module +from borgmatic.config import legacy as module def test_parse_section_options_with_punctuation_should_return_section_options(): diff --git a/borgmatic/tests/unit/config/__init__.py b/borgmatic/tests/unit/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/borgmatic/tests/unit/test_config.py b/borgmatic/tests/unit/config/test_legacy.py similarity index 99% rename from borgmatic/tests/unit/test_config.py rename to borgmatic/tests/unit/config/test_legacy.py index 01e21bb0f..9e95cac6c 100644 --- a/borgmatic/tests/unit/test_config.py +++ b/borgmatic/tests/unit/config/test_legacy.py @@ -3,7 +3,7 @@ from collections import OrderedDict from flexmock import flexmock import pytest -from borgmatic import config as module +from borgmatic.config import legacy as module def test_option_should_create_config_option(): diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 1d146ba60..3c2aa1c71 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -1,11 +1,11 @@ from collections import OrderedDict from subprocess import STDOUT +import sys import os from flexmock import flexmock from borgmatic import borg as module -from borgmatic.tests.builtins import builtins_mock from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -389,7 +389,7 @@ def test_check_archives_should_call_borg_with_parameters(): ) insert_platform_mock() insert_datetime_mock() - builtins_mock().should_receive('open').and_return(stdout) + flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) flexmock(module.os).should_receive('devnull') module.check_archives( @@ -464,7 +464,7 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param ) insert_platform_mock() insert_datetime_mock() - builtins_mock().should_receive('open').and_return(stdout) + flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) flexmock(module.os).should_receive('devnull') module.check_archives( diff --git a/sample/config.yaml b/sample/config.yaml new file mode 100644 index 000000000..6ca27c723 --- /dev/null +++ b/sample/config.yaml @@ -0,0 +1,54 @@ +location: + # List of source directories to backup. Globs are expanded. + source_directories: + - /home + - /etc + - /var/log/syslog* + + # Stay in same file system (do not cross mount points). + #one_file_system: yes + + # Alternate Borg remote executable (defaults to "borg"): + #remote_path: borg1 + + # Path to local or remote repository. + repository: user@backupserver:sourcehostname.borg + +#storage: + # Passphrase to unlock the encryption key with. Only use on repositories + # that were initialized with passphrase/repokey encryption. + #encryption_passphrase: foo + + # Type of compression to use when creating archives. See + # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create + # for details. Defaults to no compression. + #compression: lz4 + + # Umask to be used for borg create. + #umask: 0077 + +retention: + # Retention policy for how many backups to keep in each category. See + # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for + # details. + #keep_within: 3H + #keep_hourly: 24 + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 + keep_yearly: 1 + + # When pruning, only consider archive names starting with this prefix. + #prefix: sourcehostname + +consistency: + # List of consistency checks to run: "repository", "archives", or both. + # Defaults to both. Set to "disabled" to disable all consistency checks. + # See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check + # for details. + checks: + - repository + - archives + + # Restrict the number of checked archives to the last n. + #check_last: 3 diff --git a/setup.py b/setup.py index 63b00818f..198a2d55f 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.0.3-dev' +VERSION = '1.1.0' setup( @@ -30,8 +30,14 @@ setup( obsoletes=[ 'atticmatic', ], + install_requires=( + 'pykwalify', + 'ruamel.yaml<=0.15', + 'setuptools', + ), tests_require=( 'flexmock', 'pytest', - ) + ), + include_package_data=True, )