From aecb6fcd745539dfe13d9a7e43bd6d4f2abe7f49 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 16 Jun 2022 11:35:24 -0700 Subject: [PATCH] Code style, rename command-line flag, and move new code into its own file (#546) --- NEWS | 4 ++ borgmatic/commands/arguments.py | 2 +- borgmatic/config/environment.py | 37 +++++++++++++++++++ borgmatic/config/override.py | 36 ------------------ borgmatic/config/validate.py | 6 +-- setup.py | 2 +- ...t_env_variables.py => test_environment.py} | 18 ++++----- 7 files changed, 55 insertions(+), 50 deletions(-) create mode 100644 borgmatic/config/environment.py rename tests/unit/config/{test_env_variables.py => test_environment.py} (81%) diff --git a/NEWS b/NEWS index 772480bdc..fb797d52e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.6.4.dev0 + * #546: Substitute an environment variable anywhere in a borgmatic configuration option value with + new "${MY_ENV_VAR}" syntax. + 1.6.3 * #541: Add "borgmatic list --find" flag for searching for files across multiple archives, useful for hunting down that file you accidentally deleted so you can extract it. See the documentation diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index b6fda3436..46fcae4cc 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -189,7 +189,7 @@ def make_parsers(): help='One or more configuration file options to override with specified values', ) global_group.add_argument( - '--no-env', + '--no-environment-interpolation', dest='resolve_env', action='store_false', help='Do not resolve environment variables in configuration file', diff --git a/borgmatic/config/environment.py b/borgmatic/config/environment.py new file mode 100644 index 000000000..396578be5 --- /dev/null +++ b/borgmatic/config/environment.py @@ -0,0 +1,37 @@ +import os +import re + + +_VARIABLE_PATTERN = re.compile(r'(?[A-Za-z0-9_]+)((:?-)(?P[^}]+))?\}') + + +def _resolve_string(matcher): + ''' + Get the value from environment given a matcher containing a name and an optional default value. + If the variable is not defined in environment and no default value is provided, an Error is raised. + ''' + name, default = matcher.group("name"), matcher.group("default") + out = os.getenv(name, default=default) + if out is None: + raise ValueError("Cannot find variable ${name} in environment".format(name=name)) + return out + + +def resolve_env_variables(item): + ''' + Resolves variables like or ${FOO} from given configuration with values from process environment + Supported formats: + - ${FOO} will return FOO env variable + - ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar" + + If any variable is missing in environment and no default value is provided, an Error is raised. + ''' + if isinstance(item, str): + return _VARIABLE_PATTERN.sub(_resolve_string, item) + if isinstance(item, list): + for i, subitem in enumerate(item): + item[i] = resolve_env_variables(subitem) + if isinstance(item, dict): + for key, value in item.items(): + item[key] = resolve_env_variables(value) + return item diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 2c02c0308..8596cb3e3 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -1,11 +1,7 @@ import io -import os -import re import ruamel.yaml -_VARIABLE_PATTERN = re.compile(r'(?[A-Za-z0-9_]+)((:?-)(?P[^}]+))?\}') - def set_values(config, keys, value): ''' @@ -81,35 +77,3 @@ def apply_overrides(config, raw_overrides): for (keys, value) in overrides: set_values(config, keys, value) - - -def _resolve_string(matcher): - ''' - Get the value from environment given a matcher containing a name and an optional default value. - If the variable is not defined in environment and no default value is provided, an Error is raised. - ''' - name, default = matcher.group("name"), matcher.group("default") - out = os.getenv(name, default=default) - if out is None: - raise ValueError("Cannot find variable ${name} in envivonment".format(name=name)) - return out - - -def resolve_env_variables(item): - ''' - Resolves variables like or ${FOO} from given configuration with values from process environment - Supported formats: - - ${FOO} will return FOO env variable - - ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar" - - If any variable is missing in environment and no default value is provided, an Error is raised. - ''' - if isinstance(item, str): - return _VARIABLE_PATTERN.sub(_resolve_string, item) - if isinstance(item, list): - for i, subitem in enumerate(item): - item[i] = resolve_env_variables(subitem) - if isinstance(item, dict): - for key, value in item.items(): - item[key] = resolve_env_variables(value) - return item diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 198d354b7..66746ba45 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -4,7 +4,7 @@ import jsonschema import pkg_resources import ruamel.yaml -from borgmatic.config import load, normalize, override +from borgmatic.config import environment, load, normalize, override def schema_filename(): @@ -98,10 +98,10 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) + normalize.normalize(config) override.apply_overrides(config, overrides) if resolve_env: - override.resolve_env_variables(config) - normalize.normalize(config) + environment.resolve_env_variables(config) try: validator = jsonschema.Draft7Validator(schema) diff --git a/setup.py b/setup.py index ffc8f6009..95fb605bf 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.6.3' +VERSION = '1.6.4.dev0' setup( diff --git a/tests/unit/config/test_env_variables.py b/tests/unit/config/test_environment.py similarity index 81% rename from tests/unit/config/test_env_variables.py rename to tests/unit/config/test_environment.py index e49616980..cf164baca 100644 --- a/tests/unit/config/test_env_variables.py +++ b/tests/unit/config/test_environment.py @@ -1,39 +1,39 @@ import pytest -from borgmatic.config import override as module +from borgmatic.config import environment as module def test_env(monkeypatch): - monkeypatch.setenv("MY_CUSTOM_VALUE", "foo") + monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo') config = {'key': 'Hello $MY_CUSTOM_VALUE'} module.resolve_env_variables(config) assert config == {'key': 'Hello $MY_CUSTOM_VALUE'} def test_env_braces(monkeypatch): - monkeypatch.setenv("MY_CUSTOM_VALUE", "foo") + monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo') config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} module.resolve_env_variables(config) assert config == {'key': 'Hello foo'} def test_env_default_value(monkeypatch): - monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False) + monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False) config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'} module.resolve_env_variables(config) assert config == {'key': 'Hello bar'} def test_env_unknown(monkeypatch): - monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False) + monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False) config = {'key': 'Hello ${MY_CUSTOM_VALUE}'} with pytest.raises(ValueError): module.resolve_env_variables(config) def test_env_full(monkeypatch): - monkeypatch.setenv("MY_CUSTOM_VALUE", "foo") - monkeypatch.delenv("MY_CUSTOM_VALUE2", raising=False) + monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo') + monkeypatch.delenv('MY_CUSTOM_VALUE2', raising=False) config = { 'key': 'Hello $MY_CUSTOM_VALUE is not resolved', 'dict': { @@ -62,8 +62,8 @@ def test_env_full(monkeypatch): 'anotherdict': { 'key': 'My foo here', 'other': 'foo', - 'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',], + 'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'], }, }, - 'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',], + 'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'], }