From f8bc67be8de64326321424a9a8ef52d9106113f0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 21 Oct 2019 15:17:47 -0700 Subject: [PATCH] Config generation support for sequences of maps, needed for database dump hooks (#225). --- .gitignore | 1 + NEWS | 3 + borgmatic/config/convert.py | 4 +- borgmatic/config/generate.py | 97 ++++++++++++++++++----- setup.py | 2 +- tests/integration/config/test_generate.py | 37 +++++++-- tests/unit/config/test_generate.py | 40 +++++++++- 7 files changed, 155 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 20109913d..f5883b582 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .coverage .pytest_cache .tox +__pycache__ build/ dist/ pip-wheel-metadata/ diff --git a/NEWS b/NEWS index 97b952e3f..49e7ea689 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.3.27.dev0 + * #225: Database dump/restore hooks for PostgreSQL (incomplete as of right now). + 1.3.26 * #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful (non-checkpoint) archives. diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 23cc08ba1..83384ea5a 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -54,10 +54,10 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') # Add comments to each section, and then add comments to the fields in each section. - generate.add_comments_to_configuration(destination_config, schema) + generate.add_comments_to_configuration_map(destination_config, schema) for section_name, section_config in destination_config.items(): - generate.add_comments_to_configuration( + generate.add_comments_to_configuration_map( section_config, schema['map'][section_name], indent=generate.INDENT ) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 179e66075..c0399ed2c 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -1,8 +1,11 @@ +import io import os +import re from ruamel import yaml INDENT = 4 +SEQUENCE_INDENT = 2 def _insert_newline_before_comment(config, field_name): @@ -15,7 +18,7 @@ def _insert_newline_before_comment(config, field_name): ) -def _schema_to_sample_configuration(schema, level=0): +def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): ''' Given a loaded configuration schema, generate and return sample config for it. Include comments for each section based on the schema "desc" description. @@ -24,14 +27,29 @@ def _schema_to_sample_configuration(schema, level=0): if example is not None: return example - config = yaml.comments.CommentedMap( - [ - (section_name, _schema_to_sample_configuration(section_schema, level + 1)) - for section_name, section_schema in schema['map'].items() - ] - ) - - add_comments_to_configuration(config, schema, indent=(level * INDENT)) + if 'seq' in schema: + config = yaml.comments.CommentedSeq( + [ + _schema_to_sample_configuration(item_schema, level, parent_is_sequence=True) + for item_schema in schema['seq'] + ] + ) + add_comments_to_configuration_sequence( + config, schema, indent=(level * INDENT) + SEQUENCE_INDENT + ) + elif 'map' in schema: + config = yaml.comments.CommentedMap( + [ + (section_name, _schema_to_sample_configuration(section_schema, level + 1)) + for section_name, section_schema in schema['map'].items() + ] + ) + indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) + add_comments_to_configuration_map( + config, schema, indent=indent, skip_first=parent_is_sequence + ) + else: + raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema)) return config @@ -42,13 +60,12 @@ def _comment_out_line(line): if not stripped_line or stripped_line.startswith('#'): return line - # Comment out the names of optional sections. - one_indent = ' ' * INDENT - if not line.startswith(one_indent): - return '# ' + line + # Comment out the names of optional sections, inserting the '#' after any indent for aesthetics. + matches = re.match(r'(\s*)', line) + indent_spaces = matches.group(0) if matches else '' + count_indent_spaces = len(indent_spaces) - # Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics. - return '# '.join((one_indent, line[INDENT:])) + return '# '.join((indent_spaces, line[count_indent_spaces:])) REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'} @@ -90,7 +107,12 @@ def _render_configuration(config): ''' Given a config data structure of nested OrderedDicts, render the config as YAML and return it. ''' - return yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT) + dumper = yaml.YAML() + dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT) + rendered = io.StringIO() + dumper.dump(config, rendered) + + return rendered.getvalue() def write_configuration(config_filename, rendered_config, mode=0o600): @@ -112,13 +134,49 @@ def write_configuration(config_filename, rendered_config, mode=0o600): os.chmod(config_filename, mode) -def add_comments_to_configuration(config, schema, indent=0): +def add_comments_to_configuration_sequence(config, schema, indent=0): + ''' + If the given config sequence's items are maps, then mine the schema for the description of the + map's first item, and slap that atop the sequence. Indent the comment the given number of + characters. + + Doing this for sequences of maps results in nice comments that look like: + + ``` + things: + # First key description. Added by this function. + - key: foo + # Second key description. Added by add_comments_to_configuration_map(). + other: bar + ``` + ''' + if 'map' not in schema['seq'][0]: + return + + for field_name in config[0].keys(): + field_schema = schema['seq'][0]['map'].get(field_name, {}) + description = field_schema.get('desc') + + # No description to use? Skip it. + if not field_schema or not description: + return + + config[0].yaml_set_start_comment(description, indent=indent) + + # We only want the first key's description here, as the rest of the keys get commented by + # add_comments_to_configuration_map(). + return + + +def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False): ''' Using descriptions from a schema as a source, add those descriptions as comments to the given - config before each field. This function only adds comments for the top-most config map level. - Indent the comment the given number of characters. + config mapping, before each field. Indent the comment the given number of characters. ''' for index, field_name in enumerate(config.keys()): + if skip_first and index == 0: + continue + field_schema = schema['map'].get(field_name, {}) description = field_schema.get('desc') @@ -127,6 +185,7 @@ def add_comments_to_configuration(config, schema, indent=0): continue config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent) + if index > 0: _insert_newline_before_comment(config, field_name) diff --git a/setup.py b/setup.py index f1dae3031..e43926367 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.3.26' +VERSION = '1.3.27.dev0' setup( diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 828f6c5dc..7065a6022 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -40,6 +40,12 @@ def test_comment_out_line_comments_indented_option(): assert module._comment_out_line(line) == ' # enabled: true' +def test_comment_out_line_comments_twice_indented_option(): + line = ' - item' + + assert module._comment_out_line(line) == ' # - item' + + def test_comment_out_optional_configuration_comments_optional_config_only(): flexmock(module)._comment_out_line = lambda line: '# ' + line config = ''' @@ -74,10 +80,10 @@ location: assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip() -def test_render_configuration_does_not_raise(): - flexmock(module.yaml).should_receive('round_trip_dump') +def test_render_configuration_converts_configuration_to_yaml_string(): + yaml_string = module._render_configuration({'foo': 'bar'}) - module._render_configuration({}) + assert yaml_string == 'foo: bar\n' def test_write_configuration_does_not_raise(): @@ -107,12 +113,33 @@ def test_write_configuration_with_already_existing_directory_does_not_raise(): module.write_configuration('config.yaml', 'config: yaml') -def test_add_comments_to_configuration_does_not_raise(): +def test_add_comments_to_configuration_sequence_of_strings_does_not_raise(): + config = module.yaml.comments.CommentedSeq(['foo', 'bar']) + schema = {'seq': [{'type': 'str'}]} + + module.add_comments_to_configuration_sequence(config, schema) + + +def test_add_comments_to_configuration_sequence_of_maps_does_not_raise(): + config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])]) + schema = {'seq': [{'map': {'foo': {'desc': 'yo'}}}]} + + module.add_comments_to_configuration_sequence(config, schema) + + +def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise(): + config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])]) + schema = {'seq': [{'map': {'foo': {}}}]} + + module.add_comments_to_configuration_sequence(config, schema) + + +def test_add_comments_to_configuration_map_does_not_raise(): # Ensure that it can deal with fields both in the schema and missing from the schema. config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}} - module.add_comments_to_configuration(config, schema) + module.add_comments_to_configuration_map(config, schema) def test_generate_sample_configuration_does_not_raise(): diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 644c295ec..aea5f1004 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -1,13 +1,14 @@ from collections import OrderedDict +import pytest from flexmock import flexmock from borgmatic.config import generate as module -def test_schema_to_sample_configuration_generates_config_with_examples(): +def test_schema_to_sample_configuration_generates_config_map_with_examples(): flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) - flexmock(module).should_receive('add_comments_to_configuration') + flexmock(module).should_receive('add_comments_to_configuration_map') schema = { 'map': OrderedDict( [ @@ -35,3 +36,38 @@ def test_schema_to_sample_configuration_generates_config_with_examples(): ('section2', OrderedDict([('field2', 'Example 2'), ('field3', 'Example 3')])), ] ) + + +def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example(): + flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list) + flexmock(module).should_receive('add_comments_to_configuration_sequence') + schema = {'seq': [{'type': 'str'}], 'example': ['hi']} + + config = module._schema_to_sample_configuration(schema) + + assert config == ['hi'] + + +def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_examples(): + flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list) + flexmock(module).should_receive('add_comments_to_configuration_sequence') + schema = { + 'seq': [ + { + 'map': OrderedDict( + [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + ) + } + ] + } + + config = module._schema_to_sample_configuration(schema) + + assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])] + + +def test_schema_to_sample_configuration_with_unsupported_schema_raises(): + schema = {'gobbledygook': [{'type': 'not-your'}]} + + with pytest.raises(ValueError): + module._schema_to_sample_configuration(schema)