Code style, rename command-line flag, and move new code into its own file (#546)

This commit is contained in:
Dan Helfman 2022-06-16 11:35:24 -07:00
parent ea45f6c4c8
commit aecb6fcd74
7 changed files with 55 additions and 50 deletions

4
NEWS
View File

@ -1,3 +1,7 @@
1.6.4.dev0
* #546: Substitute an environment variable anywhere in a borgmatic configuration option value with
new "${MY_ENV_VAR}" syntax.
1.6.3
* #541: Add "borgmatic list --find" flag for searching for files across multiple archives, useful
for hunting down that file you accidentally deleted so you can extract it. See the documentation

View File

@ -189,7 +189,7 @@ def make_parsers():
help='One or more configuration file options to override with specified values',
)
global_group.add_argument(
'--no-env',
'--no-environment-interpolation',
dest='resolve_env',
action='store_false',
help='Do not resolve environment variables in configuration file',

View File

@ -0,0 +1,37 @@
import os
import re
_VARIABLE_PATTERN = re.compile(r'(?<!\\)\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\}')
def _resolve_string(matcher):
'''
Get the value from environment given a matcher containing a name and an optional default value.
If the variable is not defined in environment and no default value is provided, an Error is raised.
'''
name, default = matcher.group("name"), matcher.group("default")
out = os.getenv(name, default=default)
if out is None:
raise ValueError("Cannot find variable ${name} in environment".format(name=name))
return out
def resolve_env_variables(item):
'''
Resolves variables like or ${FOO} from given configuration with values from process environment
Supported formats:
- ${FOO} will return FOO env variable
- ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
If any variable is missing in environment and no default value is provided, an Error is raised.
'''
if isinstance(item, str):
return _VARIABLE_PATTERN.sub(_resolve_string, item)
if isinstance(item, list):
for i, subitem in enumerate(item):
item[i] = resolve_env_variables(subitem)
if isinstance(item, dict):
for key, value in item.items():
item[key] = resolve_env_variables(value)
return item

View File

@ -1,11 +1,7 @@
import io
import os
import re
import ruamel.yaml
_VARIABLE_PATTERN = re.compile(r'(?<!\\)\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\}')
def set_values(config, keys, value):
'''
@ -81,35 +77,3 @@ def apply_overrides(config, raw_overrides):
for (keys, value) in overrides:
set_values(config, keys, value)
def _resolve_string(matcher):
'''
Get the value from environment given a matcher containing a name and an optional default value.
If the variable is not defined in environment and no default value is provided, an Error is raised.
'''
name, default = matcher.group("name"), matcher.group("default")
out = os.getenv(name, default=default)
if out is None:
raise ValueError("Cannot find variable ${name} in envivonment".format(name=name))
return out
def resolve_env_variables(item):
'''
Resolves variables like or ${FOO} from given configuration with values from process environment
Supported formats:
- ${FOO} will return FOO env variable
- ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
If any variable is missing in environment and no default value is provided, an Error is raised.
'''
if isinstance(item, str):
return _VARIABLE_PATTERN.sub(_resolve_string, item)
if isinstance(item, list):
for i, subitem in enumerate(item):
item[i] = resolve_env_variables(subitem)
if isinstance(item, dict):
for key, value in item.items():
item[key] = resolve_env_variables(value)
return item

View File

@ -4,7 +4,7 @@ import jsonschema
import pkg_resources
import ruamel.yaml
from borgmatic.config import load, normalize, override
from borgmatic.config import environment, load, normalize, override
def schema_filename():
@ -98,10 +98,10 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
raise Validation_error(config_filename, (str(error),))
normalize.normalize(config)
override.apply_overrides(config, overrides)
if resolve_env:
override.resolve_env_variables(config)
normalize.normalize(config)
environment.resolve_env_variables(config)
try:
validator = jsonschema.Draft7Validator(schema)

View File

@ -1,6 +1,6 @@
from setuptools import find_packages, setup
VERSION = '1.6.3'
VERSION = '1.6.4.dev0'
setup(

View File

@ -1,39 +1,39 @@
import pytest
from borgmatic.config import override as module
from borgmatic.config import environment as module
def test_env(monkeypatch):
monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
config = {'key': 'Hello $MY_CUSTOM_VALUE'}
module.resolve_env_variables(config)
assert config == {'key': 'Hello $MY_CUSTOM_VALUE'}
def test_env_braces(monkeypatch):
monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
module.resolve_env_variables(config)
assert config == {'key': 'Hello foo'}
def test_env_default_value(monkeypatch):
monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False)
config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'}
module.resolve_env_variables(config)
assert config == {'key': 'Hello bar'}
def test_env_unknown(monkeypatch):
monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False)
config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
with pytest.raises(ValueError):
module.resolve_env_variables(config)
def test_env_full(monkeypatch):
monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
monkeypatch.delenv("MY_CUSTOM_VALUE2", raising=False)
monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
monkeypatch.delenv('MY_CUSTOM_VALUE2', raising=False)
config = {
'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
'dict': {
@ -62,8 +62,8 @@ def test_env_full(monkeypatch):
'anotherdict': {
'key': 'My foo here',
'other': 'foo',
'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'],
},
},
'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'],
}