From f19a40ef9c479a625bd82eeb79d6a70e7793d43e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 8 Jul 2017 22:33:51 -0700 Subject: [PATCH] Basic YAML generating / validating / converting to. --- NEWS | 5 +- borgmatic/commands/__init__.py | 0 .../{command.py => commands/borgmatic.py} | 7 +- borgmatic/commands/convert_config.py | 54 +++++++++++ borgmatic/config/convert.py | 41 +++++++++ borgmatic/config/generate.py | 90 +++++++++++++++++++ borgmatic/config/schema.yaml | 62 +++++++++++++ borgmatic/config/{yaml.py => validate.py} | 33 +++---- .../tests/integration/commands/__init__.py | 0 .../test_borgmatic.py} | 2 +- .../config/{test_yaml.py => test_validate.py} | 13 +-- borgmatic/tests/unit/config/test_convert.py | 44 +++++++++ sample/config.yaml | 6 +- setup.py | 5 +- test_requirements.txt | 3 + tox.ini | 2 +- 16 files changed, 327 insertions(+), 40 deletions(-) create mode 100644 borgmatic/commands/__init__.py rename borgmatic/{command.py => commands/borgmatic.py} (83%) create mode 100644 borgmatic/commands/convert_config.py create mode 100644 borgmatic/config/convert.py create mode 100644 borgmatic/config/generate.py rename borgmatic/config/{yaml.py => validate.py} (65%) create mode 100644 borgmatic/tests/integration/commands/__init__.py rename borgmatic/tests/integration/{test_command.py => commands/test_borgmatic.py} (97%) rename borgmatic/tests/integration/config/{test_yaml.py => test_validate.py} (88%) create mode 100644 borgmatic/tests/unit/config/test_convert.py diff --git a/NEWS b/NEWS index b5d4216..3d16ba5 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ -1.1.0 +1.1.0.dev0 + * Switched config file format to YAML. Run convert-borgmatic-config to upgrade. + * Dropped Python 2 support. Now Python 3 only. * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. - * Dropped Python 2 support. Now Python 3 only. * Added logo. 1.0.3 diff --git a/borgmatic/commands/__init__.py b/borgmatic/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/borgmatic/command.py b/borgmatic/commands/borgmatic.py similarity index 83% rename from borgmatic/command.py rename to borgmatic/commands/borgmatic.py index cba6ef3..c817976 100644 --- a/borgmatic/command.py +++ b/borgmatic/commands/borgmatic.py @@ -5,7 +5,7 @@ from subprocess import CalledProcessError import sys from borgmatic import borg -from borgmatic.config.yaml import parse_configuration, schema_filename +from borgmatic.config.validate import parse_configuration, schema_filename DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' @@ -14,9 +14,8 @@ DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' def parse_arguments(*arguments): ''' - Given the name of the command with which this script was invoked and command-line arguments, - parse the arguments and return them as an ArgumentParser instance. Use the command name to - determine the default configuration and excludes paths. + Given command-line arguments with which this script was invoked, parse the arguments and return + them as an ArgumentParser instance. ''' parser = ArgumentParser() parser.add_argument( diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py new file mode 100644 index 0000000..8f645c0 --- /dev/null +++ b/borgmatic/commands/convert_config.py @@ -0,0 +1,54 @@ +from __future__ import print_function +from argparse import ArgumentParser +import os +from subprocess import CalledProcessError +import sys + +from ruamel import yaml + +from borgmatic import borg +from borgmatic.config import convert, generate, legacy, validate + + +DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config' +DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' +DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' + + +def parse_arguments(*arguments): + ''' + Given command-line arguments with which this script was invoked, parse the arguments and return + them as an ArgumentParser instance. + ''' + parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.') + parser.add_argument( + '-s', '--source', + dest='source_filename', + default=DEFAULT_SOURCE_CONFIG_FILENAME, + help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME), + ) + parser.add_argument( + '-d', '--destination', + dest='destination_filename', + default=DEFAULT_DESTINATION_CONFIG_FILENAME, + help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME), + ) + + return parser.parse_args(arguments) + + +def main(): + try: + args = parse_arguments(*sys.argv[1:]) + source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT) + schema = yaml.round_trip_load(open(validate.schema_filename()).read()) + + destination_config = convert.convert_legacy_parsed_config(source_config, schema) + + generate.write_configuration(args.destination_filename, destination_config) + + # TODO: As a backstop, check that the written config can actually be read and parsed, and + # that it matches the destination config data structure that was written. + except (ValueError, OSError) as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py new file mode 100644 index 0000000..72b9ba3 --- /dev/null +++ b/borgmatic/config/convert.py @@ -0,0 +1,41 @@ +from ruamel import yaml + +from borgmatic.config import generate + + +def _convert_section(source_section_config, section_schema): + ''' + Given a legacy Parsed_config instance for a single section, convert it to its corresponding + yaml.comments.CommentedMap representation in preparation for actual serialization to YAML. + + Additionally, use the section schema as a source of helpful comments to include within the + returned CommentedMap. + ''' + destination_section_config = yaml.comments.CommentedMap(source_section_config) + generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT) + + return destination_section_config + + +def convert_legacy_parsed_config(source_config, schema): + ''' + Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its + corresponding yaml.comments.CommentedMap representation in preparation for actual serialization + to YAML. + + Additionally, use the given schema as a source of helpful comments to include within the + returned CommentedMap. + ''' + destination_config = yaml.comments.CommentedMap([ + (section_name, _convert_section(section_config, schema['map'][section_name])) + for section_name, section_config in source_config._asdict().items() + ]) + + destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ') + + if source_config.consistency['checks']: + destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') + + generate.add_comments_to_configuration(destination_config, schema) + + return destination_config diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py new file mode 100644 index 0000000..cf4861b --- /dev/null +++ b/borgmatic/config/generate.py @@ -0,0 +1,90 @@ +from collections import OrderedDict + +from ruamel import yaml + + +INDENT = 4 + + +def write_configuration(config_filename, config): + ''' + Given a target config filename and a config data structure of nested OrderedDicts, write out the + config to file as YAML. + ''' + with open(config_filename, 'w') as config_file: + config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)) + + +def _insert_newline_before_comment(config, field_name): + ''' + Using some ruamel.yaml black magic, insert a blank line in the config right befor the given + field and its comments. + ''' + config.ca.items[field_name][1].insert( + 0, + yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None), + ) + + +def add_comments_to_configuration(config, schema, indent=0): + ''' + Using descriptions from a schema as a source, add those descriptions as comments to the given + config before each field. This function only adds comments for the top-most config map level. + Indent the comment the given number of characters. + ''' + for index, field_name in enumerate(config.keys()): + field_schema = schema['map'].get(field_name, {}) + description = field_schema.get('desc') + + # No description to use? Skip it. + if not schema or not description: + continue + + config.yaml_set_comment_before_after_key( + key=field_name, + before=description, + indent=indent, + ) + if index > 0: + _insert_newline_before_comment(config, field_name) + + +def _section_schema_to_sample_configuration(section_schema): + ''' + Given the schema for a particular config section, generate and return sample config for that + section. Include comments for each field based on the schema "desc" description. + ''' + section_config = yaml.comments.CommentedMap([ + (field_name, field_schema['example']) + for field_name, field_schema in section_schema['map'].items() + ]) + + add_comments_to_configuration(section_config, section_schema, indent=INDENT) + + return section_config + + +def _schema_to_sample_configuration(schema): + ''' + Given a loaded configuration schema, generate and return sample config for it. Include comments + for each section based on the schema "desc" description. + ''' + config = yaml.comments.CommentedMap([ + (section_name, _section_schema_to_sample_configuration(section_schema)) + for section_name, section_schema in schema['map'].items() + ]) + + add_comments_to_configuration(config, schema) + + return config + + +def generate_sample_configuration(config_filename, schema_filename): + ''' + Given a target config filename and the path to a schema filename in pykwalify YAML schema + format, write out a sample configuration file based on that schema. + ''' + schema = yaml.round_trip_load(open(schema_filename)) + config = _schema_to_sample_configuration(schema) + + write_configuration(config_filename, config) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 83c50bb..2cc2f4e 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1,48 +1,110 @@ +name: Borgmatic configuration file schema map: location: + desc: | + Where to look for files to backup, and where to store those backups. See + https://borgbackup.readthedocs.io/en/stable/quickstart.html and + https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details. required: True map: source_directories: required: True seq: - type: scalar + desc: List of source directories to backup. Globs are expanded. + example: + - /home + - /etc + - /var/log/syslog* one_file_system: type: bool + desc: Stay in same file system (do not cross mount points). + example: yes remote_path: type: scalar + desc: Alternate Borg remote executable. Defaults to "borg". + example: borg1 repository: required: True type: scalar + desc: Path to local or remote repository. + example: user@backupserver:sourcehostname.borg storage: + desc: | + Repository storage options. See + https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and + https://borgbackup.readthedocs.io/en/stable/usage.html#environment-variables for details. map: encryption_passphrase: type: scalar + desc: | + Passphrase to unlock the encryption key with. Only use on repositories that were + initialized with passphrase/repokey encryption. Quote the value if it contains + punctuation, so it parses correctly. And backslash any quote or backslash + literals as well. + example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" compression: type: scalar + desc: | + Type of compression to use when creating archives. See + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details. + Defaults to no compression. + example: lz4 umask: type: scalar + desc: Umask to be used for borg create. + example: 0077 retention: + desc: | + Retention policy for how many backups to keep in each category. See + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. map: keep_within: type: scalar + desc: Keep all archives within this time interval. + example: 3H keep_hourly: type: int + desc: Number of hourly archives to keep. + example: 24 keep_daily: type: int + desc: Number of daily archives to keep. + example: 7 keep_weekly: type: int + desc: Number of weekly archives to keep. + example: 4 keep_monthly: type: int + desc: Number of monthly archives to keep. + example: 6 keep_yearly: type: int + desc: Number of yearly archives to keep. + example: 1 prefix: type: scalar + desc: When pruning, only consider archive names starting with this prefix. + example: sourcehostname consistency: + desc: | + Consistency checks to run after backups. See + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. map: checks: seq: - type: str enum: ['repository', 'archives', 'disabled'] unique: True + desc: | + List of consistency checks to run: "repository", "archives", or both. Defaults + to both. Set to "disabled" to disable all consistency checks. See + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. + example: + - repository + - archives check_last: type: int + desc: Restrict the number of checked archives to the last n. + example: 3 diff --git a/borgmatic/config/yaml.py b/borgmatic/config/validate.py similarity index 65% rename from borgmatic/config/yaml.py rename to borgmatic/config/validate.py index 4ff01b9..5dae6f3 100644 --- a/borgmatic/config/yaml.py +++ b/borgmatic/config/validate.py @@ -5,7 +5,7 @@ import warnings import pkg_resources import pykwalify.core import pykwalify.errors -import ruamel.yaml.error +from ruamel import yaml def schema_filename(): @@ -38,20 +38,18 @@ def parse_configuration(config_filename, schema_filename): Raise FileNotFoundError if the file does not exist, PermissionError if the user does not have permissions to read the file, or Validation_error if the config does not match the schema. ''' - warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning) - logging.getLogger('pykwalify').setLevel(logging.CRITICAL) - try: - validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename]) - except pykwalify.errors.CoreError as error: - if 'do not exists on disk' in str(error): - raise FileNotFoundError("No such file or directory: '{}'".format(config_filename)) - if 'Unable to load any data' in str(error): - # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful. - # So reach back to the originating exception from ruamel.yaml for something more useful. - raise Validation_error(config_filename, (error.__context__,)) - raise + schema = yaml.round_trip_load(open(schema_filename)) + except yaml.error.YAMLError as error: + raise Validation_error(config_filename, (str(error),)) + # pykwalify gets angry if the example field is not a string. So rather than bend to its will, + # simply remove all examples before passing the schema to pykwalify. + for section_name, section_schema in schema['map'].items(): + for field_name, field_schema in section_schema['map'].items(): + field_schema.pop('example') + + validator = pykwalify.core.Core(source_file=config_filename, schema_data=schema) parsed_result = validator.validate(raise_exception=False) if validator.validation_errors: @@ -73,12 +71,3 @@ def display_validation_error(validation_error): for error in validation_error.error_messages: print(error, file=sys.stderr) - - -# FOR TESTING -if __name__ == '__main__': - try: - configuration = parse_configuration('sample/config.yaml', schema_filename()) - print(configuration) - except Validation_error as error: - display_validation_error(error) diff --git a/borgmatic/tests/integration/commands/__init__.py b/borgmatic/tests/integration/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/borgmatic/tests/integration/test_command.py b/borgmatic/tests/integration/commands/test_borgmatic.py similarity index 97% rename from borgmatic/tests/integration/test_command.py rename to borgmatic/tests/integration/commands/test_borgmatic.py index c1e311c..69c2468 100644 --- a/borgmatic/tests/integration/test_command.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -4,7 +4,7 @@ import sys from flexmock import flexmock import pytest -from borgmatic import command as module +from borgmatic.commands import borgmatic as module def test_parse_arguments_with_no_arguments_uses_defaults(): diff --git a/borgmatic/tests/integration/config/test_yaml.py b/borgmatic/tests/integration/config/test_validate.py similarity index 88% rename from borgmatic/tests/integration/config/test_yaml.py rename to borgmatic/tests/integration/config/test_validate.py index ebd33a3..4769551 100644 --- a/borgmatic/tests/integration/config/test_yaml.py +++ b/borgmatic/tests/integration/config/test_validate.py @@ -6,7 +6,7 @@ import os from flexmock import flexmock import pytest -from borgmatic.config import yaml as module +from borgmatic.config import validate as module def test_schema_filename_returns_plausable_path(): @@ -18,13 +18,13 @@ def test_schema_filename_returns_plausable_path(): def mock_config_and_schema(config_yaml): ''' Set up mocks for the config config YAML string and the default schema so that pykwalify consumes - them when parsing the configuration. This is a little brittle in that it's relying on pykwalify - to open() the respective files in a particular order. + them when parsing the configuration. This is a little brittle in that it's relying on the code + under test to open() the respective files in a particular order. ''' - config_stream = io.StringIO(config_yaml) schema_stream = open(module.schema_filename()) + config_stream = io.StringIO(config_yaml) builtins = flexmock(sys.modules['builtins']).should_call('open').mock - builtins.should_receive('open').and_return(config_stream).and_return(schema_stream) + builtins.should_receive('open').and_return(schema_stream).and_return(config_stream) flexmock(os.path).should_receive('exists').and_return(True) @@ -87,7 +87,8 @@ def test_parse_configuration_raises_for_missing_config_file(): def test_parse_configuration_raises_for_missing_schema_file(): mock_config_and_schema('') - flexmock(os.path).should_receive('exists').with_args('schema.yaml').and_return(False) + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError) with pytest.raises(FileNotFoundError): module.parse_configuration('config.yaml', 'schema.yaml') diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py new file mode 100644 index 0000000..39fc385 --- /dev/null +++ b/borgmatic/tests/unit/config/test_convert.py @@ -0,0 +1,44 @@ +from collections import defaultdict, OrderedDict, namedtuple + +from borgmatic.config import convert as module + + +Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency')) + + +def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): + source_config = Parsed_config( + location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]), + storage=OrderedDict([('encryption_passphrase', 'supersecret')]), + retention=OrderedDict([('keep_daily', 7)]), + consistency=OrderedDict([('checks', 'repository')]), + ) + schema = {'map': defaultdict(lambda: {'map': {}})} + + destination_config = module.convert_legacy_parsed_config(source_config, schema) + + assert destination_config == OrderedDict([ + ('location', OrderedDict([('source_directories', ['/home']), ('repository', 'hostname.borg')])), + ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])), + ('retention', OrderedDict([('keep_daily', 7)])), + ('consistency', OrderedDict([('checks', ['repository'])])), + ]) + + +def test_convert_legacy_parsed_config_splits_space_separated_values(): + source_config = Parsed_config( + location=OrderedDict([('source_directories', '/home /etc')]), + storage=OrderedDict(), + retention=OrderedDict(), + consistency=OrderedDict([('checks', 'repository archives')]), + ) + schema = {'map': defaultdict(lambda: {'map': {}})} + + destination_config = module.convert_legacy_parsed_config(source_config, schema) + + assert destination_config == OrderedDict([ + ('location', OrderedDict([('source_directories', ['/home', '/etc'])])), + ('storage', OrderedDict()), + ('retention', OrderedDict()), + ('consistency', OrderedDict([('checks', ['repository', 'archives'])])), + ]) diff --git a/sample/config.yaml b/sample/config.yaml index 6ca27c7..daac230 100644 --- a/sample/config.yaml +++ b/sample/config.yaml @@ -16,8 +16,10 @@ location: #storage: # Passphrase to unlock the encryption key with. Only use on repositories - # that were initialized with passphrase/repokey encryption. - #encryption_passphrase: foo + # that were initialized with passphrase/repokey encryption. Quote the value + # if it contains punctuation so it parses correctly. And backslash any + # quote or backslash literals as well. + #encryption_passphrase: "foo" # Type of compression to use when creating archives. See # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create diff --git a/setup.py b/setup.py index 198a2d5..8ffeaba 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.0' +VERSION = '1.1.0.dev0' setup( @@ -24,7 +24,8 @@ setup( packages=find_packages(), entry_points={ 'console_scripts': [ - 'borgmatic = borgmatic.command:main', + 'borgmatic = borgmatic.commands.borgmatic:main', + 'convert-borgmatic-config = borgmatic.commands.convert_config:main', ] }, obsoletes=[ diff --git a/test_requirements.txt b/test_requirements.txt index 3c5f9e5..6bf66e0 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,2 +1,5 @@ flexmock==0.10.2 +pykwalify==1.6.0 pytest==2.9.1 +pytest-cov==2.5.1 +ruamel.yaml==0.15.18 diff --git a/tox.ini b/tox.ini index 579a919..7f6a375 100644 --- a/tox.ini +++ b/tox.ini @@ -5,4 +5,4 @@ skipsdist=True [testenv] usedevelop=True deps=-rtest_requirements.txt -commands = py.test borgmatic [] +commands = py.test --cov=borgmatic borgmatic []