From 0dfc935af69ccb01eecb0e14e0f6c4208dafb852 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 09:43:25 -0700 Subject: [PATCH] Merge excludes into config file format. --- borgmatic/commands/convert_config.py | 33 ++++++++++---- borgmatic/config/convert.py | 19 +++++--- borgmatic/config/schema.yaml | 12 ++++++ .../integration/commands/test_borgmatic.py | 20 +++------ .../commands/test_convert_config.py | 43 ++++++++++++++++--- 5 files changed, 95 insertions(+), 32 deletions(-) diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index 77735bf7f..2b99863b2 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -11,7 +11,6 @@ from borgmatic.config import convert, generate, legacy, validate DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config' -# TODO: Fold excludes into the YAML config file. DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' @@ -21,16 +20,27 @@ def parse_arguments(*arguments): Given command-line arguments with which this script was invoked, parse the arguments and return them as an ArgumentParser instance. ''' - parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.') + parser = ArgumentParser( + description=''' + Convert legacy INI-style borgmatic configuration and excludes files to a single YAML + configuration file. Note that this replaces any comments from the source files. + ''' + ) parser.add_argument( - '-s', '--source', - dest='source_filename', + '-s', '--source-config', + dest='source_config_filename', default=DEFAULT_SOURCE_CONFIG_FILENAME, help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME), ) parser.add_argument( - '-d', '--destination', - dest='destination_filename', + '-e', '--source-excludes', + dest='source_excludes_filename', + default=DEFAULT_SOURCE_EXCLUDES_FILENAME if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) else None, + help='Excludes filename', + ) + parser.add_argument( + '-d', '--destination-config', + dest='destination_config_filename', default=DEFAULT_DESTINATION_CONFIG_FILENAME, help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME), ) @@ -41,12 +51,17 @@ def parse_arguments(*arguments): def main(): # pragma: no cover try: args = parse_arguments(*sys.argv[1:]) - source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT) schema = yaml.round_trip_load(open(validate.schema_filename()).read()) + source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT) + source_excludes = ( + open(args.source_excludes_filename).read().splitlines() + if args.source_excludes_filename + else [] + ) - destination_config = convert.convert_legacy_parsed_config(source_config, schema) + destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema) - generate.write_configuration(args.destination_filename, destination_config) + generate.write_configuration(args.destination_config_filename, destination_config) # TODO: As a backstop, check that the written config can actually be read and parsed, and # that it matches the destination config data structure that was written. diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 72b9ba314..11618e958 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -12,16 +12,15 @@ def _convert_section(source_section_config, section_schema): returned CommentedMap. ''' destination_section_config = yaml.comments.CommentedMap(source_section_config) - generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT) return destination_section_config -def convert_legacy_parsed_config(source_config, schema): +def convert_legacy_parsed_config(source_config, source_excludes, schema): ''' - Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its - corresponding yaml.comments.CommentedMap representation in preparation for actual serialization - to YAML. + Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude + patterns, convert them to a corresponding yaml.comments.CommentedMap representation in + preparation for serialization to a single YAML config file. Additionally, use the given schema as a source of helpful comments to include within the returned CommentedMap. @@ -31,11 +30,21 @@ def convert_legacy_parsed_config(source_config, schema): for section_name, section_config in source_config._asdict().items() ]) + # Split space-seperated values into actual lists, and merge in excludes. destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ') + destination_config['location']['exclude_patterns'] = source_excludes if source_config.consistency['checks']: destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') + # Add comments to each section, and then add comments to the fields in each section. generate.add_comments_to_configuration(destination_config, schema) + for section_name, section_config in destination_config.items(): + generate.add_comments_to_configuration( + section_config, + schema['map'][section_name], + indent=generate.INDENT, + ) + return destination_config diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 347dfee80..97f13d98a 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -30,6 +30,18 @@ map: type: scalar desc: Path to local or remote repository. example: user@backupserver:sourcehostname.borg + exclude_patterns: + seq: + - type: scalar + desc: | + Exclude patterns. Any paths matching these patterns are excluded from backups. + Globs are expanded. See + https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns for + details. + example: + - '*.pyc' + - /home/*/.cache + - /etc/ssl storage: desc: | Repository storage options. See diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py index 69c24686a..12afeedf0 100644 --- a/borgmatic/tests/integration/commands/test_borgmatic.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -1,5 +1,4 @@ import os -import sys from flexmock import flexmock import pytest @@ -14,7 +13,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME - assert parser.verbosity == None + assert parser.verbosity is None def test_parse_arguments_with_filename_arguments_overrides_defaults(): @@ -24,7 +23,7 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults(): assert parser.config_filename == 'myconfig' assert parser.excludes_filename == 'myexcludes' - assert parser.verbosity == None + assert parser.verbosity is None def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none(): @@ -33,8 +32,8 @@ def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_non parser = module.parse_arguments() assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME - assert parser.excludes_filename == None - assert parser.verbosity == None + assert parser.excludes_filename is None + assert parser.verbosity is None def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(): @@ -44,7 +43,7 @@ def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename( assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.excludes_filename == 'myexcludes' - assert parser.verbosity == None + assert parser.verbosity is None def test_parse_arguments_with_verbosity_flag_overrides_default(): @@ -59,11 +58,6 @@ def test_parse_arguments_with_verbosity_flag_overrides_default(): def test_parse_arguments_with_invalid_arguments_exits(): flexmock(os.path).should_receive('exists').and_return(True) - original_stderr = sys.stderr - sys.stderr = sys.stdout - try: - with pytest.raises(SystemExit): - module.parse_arguments('--posix-me-harder') - finally: - sys.stderr = original_stderr + with pytest.raises(SystemExit): + module.parse_arguments('--posix-me-harder') diff --git a/borgmatic/tests/integration/commands/test_convert_config.py b/borgmatic/tests/integration/commands/test_convert_config.py index b2bc32bd8..e126c1201 100644 --- a/borgmatic/tests/integration/commands/test_convert_config.py +++ b/borgmatic/tests/integration/commands/test_convert_config.py @@ -1,14 +1,47 @@ +import os + +from flexmock import flexmock +import pytest + from borgmatic.commands import convert_config as module def test_parse_arguments_with_no_arguments_uses_defaults(): + flexmock(os.path).should_receive('exists').and_return(True) + parser = module.parse_arguments() - assert parser.source_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME - assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME + assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME + assert parser.source_excludes_filename == module.DEFAULT_SOURCE_EXCLUDES_FILENAME + assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME + def test_parse_arguments_with_filename_arguments_overrides_defaults(): - parser = module.parse_arguments('--source', 'config', '--destination', 'config.yaml') + flexmock(os.path).should_receive('exists').and_return(True) - assert parser.source_filename == 'config' - assert parser.destination_filename == 'config.yaml' + parser = module.parse_arguments( + '--source-config', 'config', + '--source-excludes', 'excludes', + '--destination-config', 'config.yaml', + ) + + assert parser.source_config_filename == 'config' + assert parser.source_excludes_filename == 'excludes' + assert parser.destination_config_filename == 'config.yaml' + + +def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none(): + flexmock(os.path).should_receive('exists').and_return(False) + + parser = module.parse_arguments() + + assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME + assert parser.source_excludes_filename is None + assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME + + +def test_parse_arguments_with_invalid_arguments_exits(): + flexmock(os.path).should_receive('exists').and_return(True) + + with pytest.raises(SystemExit): + module.parse_arguments('--posix-me-harder')