From 97b5cd089d714d3ea542520429b88f95608c4ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20MB?= Date: Thu, 16 Jun 2022 18:52:54 +0200 Subject: [PATCH] Allow environment variable resolution in configuration file - all string fields containing an environment variable like ${FOO} will be resolved - supported format ${FOO}, ${FOO:-bar} and ${FOO-bar} to allow default values if variable is not present in environment - add --no-env argument for CLI to disable the feature which is enabled by default Resolves: #546 --- borgmatic/commands/arguments.py | 6 +++ borgmatic/commands/borgmatic.py | 8 +-- borgmatic/config/override.py | 36 +++++++++++++ borgmatic/config/validate.py | 4 +- tests/unit/config/test_env_variables.py | 69 +++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 tests/unit/config/test_env_variables.py diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 184ab007c..b6fda3436 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -188,6 +188,12 @@ def make_parsers(): action='extend', help='One or more configuration file options to override with specified values', ) + global_group.add_argument( + '--no-env', + dest='resolve_env', + action='store_false', + help='Do not resolve environment variables in configuration file', + ) global_group.add_argument( '--bash-completion', default=False, diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 4fac16558..c3c7df6fa 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -650,7 +650,7 @@ def run_actions( ) -def load_configurations(config_filenames, overrides=None): +def load_configurations(config_filenames, overrides=None, resolve_env=True): ''' Given a sequence of configuration filenames, load and validate each configuration file. Return the results as a tuple of: dict of configuration filename to corresponding parsed configuration, @@ -664,7 +664,7 @@ def load_configurations(config_filenames, overrides=None): for config_filename in config_filenames: try: configs[config_filename] = validate.parse_configuration( - config_filename, validate.schema_filename(), overrides + config_filename, validate.schema_filename(), overrides, resolve_env ) except PermissionError: logs.extend( @@ -892,7 +892,9 @@ def main(): # pragma: no cover sys.exit(0) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) - configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides) + configs, parse_logs = load_configurations( + config_filenames, global_arguments.overrides, global_arguments.resolve_env + ) any_json_flags = any( getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values() diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 8596cb3e3..2c02c0308 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -1,7 +1,11 @@ import io +import os +import re import ruamel.yaml +_VARIABLE_PATTERN = re.compile(r'(?[A-Za-z0-9_]+)((:?-)(?P[^}]+))?\}') + def set_values(config, keys, value): ''' @@ -77,3 +81,35 @@ def apply_overrides(config, raw_overrides): for (keys, value) in overrides: set_values(config, keys, value) + + +def _resolve_string(matcher): + ''' + Get the value from environment given a matcher containing a name and an optional default value. + If the variable is not defined in environment and no default value is provided, an Error is raised. + ''' + name, default = matcher.group("name"), matcher.group("default") + out = os.getenv(name, default=default) + if out is None: + raise ValueError("Cannot find variable ${name} in envivonment".format(name=name)) + return out + + +def resolve_env_variables(item): + ''' + Resolves variables like or ${FOO} from given configuration with values from process environment + Supported formats: + - ${FOO} will return FOO env variable + - ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar" + + If any variable is missing in environment and no default value is provided, an Error is raised. + ''' + if isinstance(item, str): + return _VARIABLE_PATTERN.sub(_resolve_string, item) + if isinstance(item, list): + for i, subitem in enumerate(item): + item[i] = resolve_env_variables(subitem) + if isinstance(item, dict): + for key, value in item.items(): + item[key] = resolve_env_variables(value) + return item diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 156b4a6b7..198d354b7 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -79,7 +79,7 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration(config_filename, schema_filename, overrides=None): +def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML rendition of JSON Schema format, a sequence of configuration file override strings in the form @@ -99,6 +99,8 @@ def parse_configuration(config_filename, schema_filename, overrides=None): raise Validation_error(config_filename, (str(error),)) override.apply_overrides(config, overrides) + if resolve_env: + override.resolve_env_variables(config) normalize.normalize(config) try: diff --git a/tests/unit/config/test_env_variables.py b/tests/unit/config/test_env_variables.py new file mode 100644 index 000000000..e49616980 --- /dev/null +++ b/tests/unit/config/test_env_variables.py @@ -0,0 +1,69 @@ +import pytest + +from borgmatic.config import override as module + + +def test_env(monkeypatch): + monkeypatch.setenv("MY_CUSTOM_VALUE", "foo") + config = {'key': 'Hello $MY_CUSTOM_VALUE'} + module.resolve_env_variables(config) + assert config == {'key': 'Hello $MY_CUSTOM_VALUE'} + + +def test_env_braces(monkeypatch): + monkeypatch.setenv("MY_CUSTOM_VALUE", "foo") + config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} + module.resolve_env_variables(config) + assert config == {'key': 'Hello foo'} + + +def test_env_default_value(monkeypatch): + monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False) + config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'} + module.resolve_env_variables(config) + assert config == {'key': 'Hello bar'} + + +def test_env_unknown(monkeypatch): + monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False) + config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} + with pytest.raises(ValueError): + module.resolve_env_variables(config) + + +def test_env_full(monkeypatch): + monkeypatch.setenv("MY_CUSTOM_VALUE", "foo") + monkeypatch.delenv("MY_CUSTOM_VALUE2", raising=False) + config = { + 'key': 'Hello $MY_CUSTOM_VALUE is not resolved', + 'dict': { + 'key': 'value', + 'anotherdict': { + 'key': 'My ${MY_CUSTOM_VALUE} here', + 'other': '${MY_CUSTOM_VALUE}', + 'list': [ + '/home/${MY_CUSTOM_VALUE}/.local', + '/var/log/', + '/home/${MY_CUSTOM_VALUE2:-bar}/.config', + ], + }, + }, + 'list': [ + '/home/${MY_CUSTOM_VALUE}/.local', + '/var/log/', + '/home/${MY_CUSTOM_VALUE2-bar}/.config', + ], + } + module.resolve_env_variables(config) + assert config == { + 'key': 'Hello $MY_CUSTOM_VALUE is not resolved', + 'dict': { + 'key': 'value', + 'anotherdict': { + 'key': 'My foo here', + 'other': 'foo', + 'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',], + }, + }, + 'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',], + }