From f787dfe809561cb6d27005b43b9d58bd958bb818 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 17 Dec 2019 11:46:27 -0800 Subject: [PATCH] Override particular configuration options from the command-line via "--override" flag (#268). --- NEWS | 5 ++ borgmatic/commands/arguments.py | 7 ++ borgmatic/commands/borgmatic.py | 6 +- borgmatic/config/override.py | 71 ++++++++++++++++++ borgmatic/config/validate.py | 13 ++-- docs/how-to/make-per-application-backups.md | 34 +++++++++ setup.py | 2 +- tests/integration/config/test_override.py | 40 ++++++++++ tests/integration/config/test_validate.py | 27 +++++++ tests/unit/config/test_override.py | 82 +++++++++++++++++++++ 10 files changed, 278 insertions(+), 9 deletions(-) create mode 100644 borgmatic/config/override.py create mode 100644 tests/integration/config/test_override.py create mode 100644 tests/unit/config/test_override.py diff --git a/NEWS b/NEWS index 61abaf453..dd1c9539c 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.4.21.dev0 + * #268: Override particular configuration options from the command-line via "--override" flag. See + the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides + 1.4.20 * Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option. * #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 738981cae..630300668 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -164,6 +164,13 @@ def parse_arguments(*unparsed_arguments): default=None, help='Write log messages to this file instead of syslog', ) + global_group.add_argument( + '--override', + metavar='SECTION.OPTION=VALUE', + nargs='+', + dest='overrides', + help='One or more configuration file options to override with specified values', + ) global_group.add_argument( '--version', dest='version', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index f3da16184..ce3719d80 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -372,7 +372,7 @@ def run_actions( yield json.loads(json_output) -def load_configurations(config_filenames): +def load_configurations(config_filenames, overrides=None): ''' Given a sequence of configuration filenames, load and validate each configuration file. Return the results as a tuple of: dict of configuration filename to corresponding parsed configuration, @@ -386,7 +386,7 @@ def load_configurations(config_filenames): for config_filename in config_filenames: try: configs[config_filename] = validate.parse_configuration( - config_filename, validate.schema_filename() + config_filename, validate.schema_filename(), overrides ) except (ValueError, OSError, validate.Validation_error) as error: logs.extend( @@ -584,7 +584,7 @@ def main(): # pragma: no cover sys.exit(0) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) - configs, parse_logs = load_configurations(config_filenames) + configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides) colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs)) try: diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py new file mode 100644 index 000000000..eb86077b7 --- /dev/null +++ b/borgmatic/config/override.py @@ -0,0 +1,71 @@ +import io + +import ruamel.yaml + + +def set_values(config, keys, value): + ''' + Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value, + descend into the hierarchy based on the keys to set the value into the right place. + ''' + if not keys: + return + + first_key = keys[0] + if len(keys) == 1: + config[first_key] = value + return + + if first_key not in config: + config[first_key] = {} + + set_values(config[first_key], keys[1:], value) + + +def convert_value_type(value): + ''' + Given a string value, determine its logical type (string, boolean, integer, etc.), and return it + converted to that type. + ''' + return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + + +def parse_overrides(raw_overrides): + ''' + Given a sequence of configuration file override strings in the form of "section.option=value", + parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For + instance, given the following raw overrides: + + ['section.my_option=value1', 'section.other_option=value2'] + + ... return this: + + ( + (('section', 'my_option'), 'value1'), + (('section', 'other_option'), 'value2'), + ) + + Raise ValueError if an override can't be parsed. + ''' + if not raw_overrides: + return () + + try: + return tuple( + (tuple(raw_keys.split('.')), convert_value_type(value)) + for raw_override in raw_overrides + for raw_keys, value in (raw_override.split('=', 1),) + ) + except ValueError: + raise ValueError('Invalid override. Make sure you use the form: SECTION.OPTION=VALUE') + + +def apply_overrides(config, raw_overrides): + ''' + Given a sequence of configuration file override strings in the form of "section.option=value" + and a configuration dict, parse each override and set it the configuration dict. + ''' + overrides = parse_overrides(raw_overrides) + + for (keys, value) in overrides: + set_values(config, keys, value) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 1e421d18f..b7d34a9f8 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -6,7 +6,7 @@ import pykwalify.core import pykwalify.errors import ruamel.yaml -from borgmatic.config import load +from borgmatic.config import load, override def schema_filename(): @@ -82,11 +82,12 @@ def remove_examples(schema): return schema -def parse_configuration(config_filename, schema_filename): +def parse_configuration(config_filename, schema_filename, overrides=None): ''' - Given the path to a config filename in YAML format and the path to a schema filename in - pykwalify YAML schema format, return the parsed configuration as a data structure of nested - dicts and lists corresponding to the schema. Example return value: + Given the path to a config filename in YAML format, the path to a schema filename in pykwalify + YAML schema format, a sequence of configuration file override strings in the form of + "section.option=value", return the parsed configuration as a data structure of nested dicts and + lists corresponding to the schema. Example return value: {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, 'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}} @@ -102,6 +103,8 @@ def parse_configuration(config_filename, schema_filename): except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) + override.apply_overrides(config, overrides) + validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema)) parsed_result = validator.validate(raise_exception=False) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 840e2a6d7..cd2574608 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -115,6 +115,40 @@ Note that this `<<` include merging syntax is only for merging in mappings directly, please see the section above about standard includes. +## Configuration overrides + +In more complex multi-application setups, you may want to override particular +borgmatic configuration file options at the time you run borgmatic. For +instance, you could reuse a common configuration file for multiple +applications, but then set the repository for each application at runtime. Or +you might want to try a variant of an option for testing purposes without +actually touching your configuration file. + +Whatever the reason, you can override borgmatic configuration options at the +command-line via the `--override` flag. Here's an example: + +```bash +borgmatic create --override location.remote_path=borg1 +``` + +What this does is load your configuration files, and for each one, disregard +the configured value for the `remote_path` option in the `location` section, +and use the value of `borg1` instead. + +Note that the value is parsed as an actual YAML string, so you can even set +list values by using brackets. For instance: + +```bash +borgmatic create --override location.repositories=[test1.borg,test2.borg] +``` + +There is not currently a way to override a single element of a list without +replacing the whole list. + +Be sure to quote your overrides if they contain spaces or other characters +that your shell may interpret. + + ## Related documentation * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/) diff --git a/setup.py b/setup.py index e61d3ac81..0d87bc595 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.4.20' +VERSION = '1.4.21.dev0' setup( diff --git a/tests/integration/config/test_override.py b/tests/integration/config/test_override.py new file mode 100644 index 000000000..cfcd33944 --- /dev/null +++ b/tests/integration/config/test_override.py @@ -0,0 +1,40 @@ +import pytest + +from borgmatic.config import override as module + + +@pytest.mark.parametrize( + 'value,expected_result', + ( + ('thing', 'thing'), + ('33', 33), + ('33b', '33b'), + ('true', True), + ('false', False), + ('[foo]', ['foo']), + ('[foo, bar]', ['foo', 'bar']), + ), +) +def test_convert_value_type_coerces_values(value, expected_result): + assert module.convert_value_type(value) == expected_result + + +def test_apply_overrides_updates_config(): + raw_overrides = [ + 'section.key=value1', + 'other_section.thing=value2', + 'section.nested.key=value3', + 'new.foo=bar', + ] + config = { + 'section': {'key': 'value', 'other': 'other_value'}, + 'other_section': {'thing': 'thing_value'}, + } + + module.apply_overrides(config, raw_overrides) + + assert config == { + 'section': {'key': 'value1', 'other': 'other_value', 'nested': {'key': 'value3'}}, + 'other_section': {'thing': 'value2'}, + 'new': {'foo': 'bar'}, + } diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 706743bc0..cbd4f5baa 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -212,3 +212,30 @@ def test_parse_configuration_raises_for_validation_error(): with pytest.raises(module.Validation_error): module.parse_configuration('config.yaml', 'schema.yaml') + + +def test_parse_configuration_applies_overrides(): + mock_config_and_schema( + ''' + location: + source_directories: + - /home + + repositories: + - hostname.borg + + local_path: borg1 + ''' + ) + + result = module.parse_configuration( + 'config.yaml', 'schema.yaml', overrides=['location.local_path=borg2'] + ) + + assert result == { + 'location': { + 'source_directories': ['/home'], + 'repositories': ['hostname.borg'], + 'local_path': 'borg2', + } + } diff --git a/tests/unit/config/test_override.py b/tests/unit/config/test_override.py new file mode 100644 index 000000000..6925ca424 --- /dev/null +++ b/tests/unit/config/test_override.py @@ -0,0 +1,82 @@ +import pytest +from flexmock import flexmock + +from borgmatic.config import override as module + + +def test_set_values_with_empty_keys_bails(): + config = {} + + module.set_values(config, keys=(), value='value') + + assert config == {} + + +def test_set_values_with_one_key_sets_it_into_config(): + config = {} + + module.set_values(config, keys=('key',), value='value') + + assert config == {'key': 'value'} + + +def test_set_values_with_one_key_overwrites_existing_key(): + config = {'key': 'old_value', 'other': 'other_value'} + + module.set_values(config, keys=('key',), value='value') + + assert config == {'key': 'value', 'other': 'other_value'} + + +def test_set_values_with_multiple_keys_creates_hierarchy(): + config = {} + + module.set_values(config, ('section', 'key'), 'value') + + assert config == {'section': {'key': 'value'}} + + +def test_set_values_with_multiple_keys_updates_hierarchy(): + config = {'section': {'other': 'other_value'}} + module.set_values(config, ('section', 'key'), 'value') + + assert config == {'section': {'key': 'value', 'other': 'other_value'}} + + +def test_parse_overrides_splits_keys_and_values(): + flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value) + raw_overrides = ['section.my_option=value1', 'section.other_option=value2'] + expected_result = ( + (('section', 'my_option'), 'value1'), + (('section', 'other_option'), 'value2'), + ) + + module.parse_overrides(raw_overrides) == expected_result + + +def test_parse_overrides_allows_value_with_equal_sign(): + flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value) + raw_overrides = ['section.option=this===value'] + expected_result = ((('section', 'option'), 'this===value'),) + + module.parse_overrides(raw_overrides) == expected_result + + +def test_parse_overrides_raises_on_missing_equal_sign(): + flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value) + raw_overrides = ['section.option'] + + with pytest.raises(ValueError): + module.parse_overrides(raw_overrides) + + +def test_parse_overrides_allows_value_with_single_key(): + flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value) + raw_overrides = ['option=value'] + expected_result = ((('option',), 'value'),) + + module.parse_overrides(raw_overrides) == expected_result + + +def test_parse_overrides_handles_empty_overrides(): + module.parse_overrides(raw_overrides=None) == ()