Fix escaped environment variable in configuration

- when an env variable is escaped in the configuration file, we expect
  not to resolve it and remove the escape char `\`

NEWS

@@ -1,3 +1,8 @@
+1.6.4.dev0
+ * #546, #382: Keep your repository passphrases and database passwords outside of borgmatic's
+   configuration file with environment variable interpolation. See the documentation for more
+   information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+
 1.6.3
  * #541: Add "borgmatic list --find" flag for searching for files across multiple archives, useful
    for hunting down that file you accidentally deleted so you can extract it. See the documentation

borgmatic/commands/arguments.py

@@ -188,6 +188,12 @@ def make_parsers():
         action='extend',
         help='One or more configuration file options to override with specified values',
     )
+    global_group.add_argument(
+        '--no-environment-interpolation',
+        dest='resolve_env',
+        action='store_false',
+        help='Do not resolve environment variables in configuration file',
+    )
     global_group.add_argument(
         '--bash-completion',
         default=False,

borgmatic/commands/borgmatic.py

@@ -650,7 +650,7 @@ def run_actions(
             )
 
 
-def load_configurations(config_filenames, overrides=None):
+def load_configurations(config_filenames, overrides=None, resolve_env=True):
     '''
     Given a sequence of configuration filenames, load and validate each configuration file. Return
     the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
@@ -664,7 +664,7 @@ def load_configurations(config_filenames, overrides=None):
     for config_filename in config_filenames:
         try:
             configs[config_filename] = validate.parse_configuration(
-                config_filename, validate.schema_filename(), overrides
+                config_filename, validate.schema_filename(), overrides, resolve_env
             )
         except PermissionError:
             logs.extend(
@@ -892,7 +892,9 @@ def main():  # pragma: no cover
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
-    configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
+    configs, parse_logs = load_configurations(
+        config_filenames, global_arguments.overrides, global_arguments.resolve_env
+    )
 
     any_json_flags = any(
         getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()

borgmatic/config/environment.py

@@ -0,0 +1,40 @@
+import os
+import re
+
+_VARIABLE_PATTERN = re.compile(r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})')
+
+
+def _resolve_string(matcher):
+    '''
+    Get the value from environment given a matcher containing a name and an optional default value.
+    If the variable is not defined in environment and no default value is provided, an Error is raised.
+    '''
+    if matcher.group('escape') is not None:
+        # in case of escaped envvar, unescape it
+        return matcher.group('variable')
+    # resolve the env var
+    name, default = matcher.group('name'), matcher.group('default')
+    out = os.getenv(name, default=default)
+    if out is None:
+        raise ValueError('Cannot find variable ${name} in environment'.format(name=name))
+    return out
+
+
+def resolve_env_variables(item):
+    '''
+    Resolves variables like or ${FOO} from given configuration with values from process environment
+    Supported formats:
+     - ${FOO} will return FOO env variable
+     - ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
+
+    If any variable is missing in environment and no default value is provided, an Error is raised.
+    '''
+    if isinstance(item, str):
+        return _VARIABLE_PATTERN.sub(_resolve_string, item)
+    if isinstance(item, list):
+        for i, subitem in enumerate(item):
+            item[i] = resolve_env_variables(subitem)
+    if isinstance(item, dict):
+        for key, value in item.items():
+            item[key] = resolve_env_variables(value)
+    return item

borgmatic/config/schema.yaml

@@ -908,13 +908,13 @@ properties:
                     topic:
                         type: string
                         description: |
-                            The topic to publish to
+                            The topic to publish to.
                             (https://ntfy.sh/docs/publish/)
                         example: topic
                     server:
                         type: string
                         description: |
-                            The address of your self-hosted ntfy.sh installation
+                            The address of your self-hosted ntfy.sh instance.
                         example: https://ntfy.your-domain.com
                     start:
                         type: object
@@ -927,17 +927,17 @@ properties:
                             message:
                                 type: string
                                 description: |
-                                    The message body to publish
+                                    The message body to publish.
                                 example: Your backups have failed.
                             priority:
                                 type: string
                                 description: |
-                                    The priority to set
+                                    The priority to set.
                                 example: urgent
                             tags:
                                 type: string
                                 description: |
-                                    Tags to attach to the message
+                                    Tags to attach to the message.
                                 example: incoming_envelope
                     finish:
                         type: object
@@ -945,22 +945,22 @@ properties:
                             title:
                                 type: string
                                 description: |
-                                    The title of the message
+                                    The title of the message.
                                 example: Ping!
                             message:
                                 type: string
                                 description: |
-                                    The message body to publish
+                                    The message body to publish.
                                 example: Your backups have failed.
                             priority:
                                 type: string
                                 description: |
-                                    The priority to set
+                                    The priority to set.
                                 example: urgent
                             tags:
                                 type: string
                                 description: |
-                                    Tags to attach to the message
+                                    Tags to attach to the message.
                                 example: incoming_envelope
                     fail:
                         type: object
@@ -968,22 +968,22 @@ properties:
                             title:
                                 type: string
                                 description: |
-                                    The title of the message
+                                    The title of the message.
                                 example: Ping!
                             message:
                                 type: string
                                 description: |
-                                    The message body to publish
+                                    The message body to publish.
                                 example: Your backups have failed.
                             priority:
                                 type: string
                                 description: |
-                                    The priority to set
+                                    The priority to set.
                                 example: urgent
                             tags:
                                 type: string
                                 description: |
-                                    Tags to attach to the message
+                                    Tags to attach to the message.
                                 example: incoming_envelope
                     states:
                         type: array

borgmatic/config/validate.py

@@ -4,7 +4,7 @@ import jsonschema
 import pkg_resources
 import ruamel.yaml
 
-from borgmatic.config import load, normalize, override
+from borgmatic.config import environment, load, normalize, override
 
 
 def schema_filename():
@@ -79,7 +79,7 @@ def apply_logical_validation(config_filename, parsed_configuration):
             )
 
 
-def parse_configuration(config_filename, schema_filename, overrides=None):
+def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True):
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
     rendition of JSON Schema format, a sequence of configuration file override strings in the form
@@ -98,8 +98,10 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
     except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
-    override.apply_overrides(config, overrides)
     normalize.normalize(config)
+    override.apply_overrides(config, overrides)
+    if resolve_env:
+        environment.resolve_env_variables(config)
 
     try:
         validator = jsonschema.Draft7Validator(schema)

docs/Dockerfile

@@ -1,14 +1,14 @@
-FROM python:3.8-alpine3.13 as borgmatic
+FROM alpine:3.16.0 as borgmatic
 
 COPY . /app
-RUN apk add --no-cache py3-ruamel.yaml py3-ruamel.yaml.clib
+RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
     && for action in init prune compact create check extract export-tar mount umount restore list info borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:15.2.1-alpine as html
+FROM node:18.4.0-alpine as html
 
 ARG ENVIRONMENT=production
 
@@ -27,7 +27,7 @@ COPY . /source
 RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.19.4-alpine
+FROM nginx:1.22.0-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -3,7 +3,7 @@ title: How to add preparation and cleanup steps to backups
 eleventyNavigation:
   key: 🧹 Add preparation and cleanup steps
   parent: How-to guides
-  order: 8
+  order: 9
 ---
 ## Preparation and cleanup hooks
 
@@ -28,7 +28,8 @@ hooks:
         - umount /some/filesystem
 ```
 
-The `before_backup` and `after_backup` hooks each run once per repository in a
+<span class="minilink minilink-addedin">New in version 1.6.0</span> The
+`before_backup` and `after_backup` hooks each run once per repository in a
 configuration file. `before_backup` hooks runs right before the `create`
 action for a particular repository, and `after_backup` hooks run afterwards,
 but not if an error occurs in a previous hook or in the backups themselves.
@@ -61,6 +62,10 @@ variables you can use here:
  * `repository`: path of the current repository as configured in the current
    borgmatic configuration file
 
+Note that you can also interpolate in [arbitrary environment
+variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).
+
+
 ## Global hooks
 
 You can also use `before_everything` and `after_everything` hooks to perform

docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md

@@ -3,7 +3,7 @@ title: How to backup to a removable drive or an intermittent server
 eleventyNavigation:
   key: 💾 Backup to a removable drive/server
   parent: How-to guides
-  order: 9
+  order: 10
 ---
 ## Occasional backups
 

docs/how-to/backup-your-databases.md

@@ -3,7 +3,7 @@ title: How to backup your databases
 eleventyNavigation:
   key: 🗄️ Backup your databases
   parent: How-to guides
-  order: 7
+  order: 8
 ---
 ## Database dump hooks
 
@@ -100,6 +100,14 @@ hooks:
         - name: all
 ```
 
+### External passwords
+
+If you don't want to keep your database passwords in your borgmatic
+configuration file, you can instead pass them in via [environment
+variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/)
+or command-line [configuration
+overrides](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides).
+
 
 ### Configuration backups
 

docs/how-to/deal-with-very-large-backups.md

@@ -3,7 +3,7 @@ title: How to deal with very large backups
 eleventyNavigation:
   key: 📏 Deal with very large backups
   parent: How-to guides
-  order: 3
+  order: 4
 ---
 ## Biggish data
 
@@ -74,8 +74,9 @@ See [Borg's check documentation](https://borgbackup.readthedocs.io/en/stable/usa
 
 ### Check frequency
 
-As of borgmatic 1.6.2, you can optionally configure checks to run on a
-periodic basis rather than every time borgmatic runs checks. For instance:
+<span class="minilink minilink-addedin">New in version 1.6.2</span> You can
+optionally configure checks to run on a periodic basis rather than every time
+borgmatic runs checks. For instance:
 
 ```yaml
 consistency:

docs/how-to/develop-on-borgmatic.md

@@ -3,7 +3,7 @@ title: How to develop on borgmatic
 eleventyNavigation:
   key: 🏗️ Develop on borgmatic
   parent: How-to guides
-  order: 12
+  order: 13
 ---
 ## Source code
 

docs/how-to/extract-a-backup.md

@@ -3,7 +3,7 @@ title: How to extract a backup
 eleventyNavigation:
   key: 📤 Extract a backup
   parent: How-to guides
-  order: 6
+  order: 7
 ---
 ## Extract
 

docs/how-to/inspect-your-backups.md

@@ -3,7 +3,7 @@ title: How to inspect your backups
 eleventyNavigation:
   key: 🔎 Inspect your backups
   parent: How-to guides
-  order: 4
+  order: 5
 ---
 ## Backup progress
 
@@ -53,10 +53,10 @@ borgmatic info
 
 ### Searching for a file
 
-Let's say you've accidentally deleted a file and want to find the backup
-archive(s) containing it. `borgmatic list` provides a `--find` flag for
-exactly this purpose (as of borgmatic 1.6.3). For instance, if you're looking
-for a `foo.txt`:
+<span class="minilink minilink-addedin">New in version 1.6.3</span> Let's say
+you've accidentally deleted a file and want to find the backup archive(s)
+containing it. `borgmatic list` provides a `--find` flag for exactly this
+purpose. For instance, if you're looking for a `foo.txt`:
 
 ```bash
 borgmatic list --find foo.txt

docs/how-to/make-backups-redundant.md

@@ -3,7 +3,7 @@ title: How to make backups redundant
 eleventyNavigation:
   key: ☁️  Make backups redundant
   parent: How-to guides
-  order: 2
+  order: 3
 ---
 ## Multiple repositories
 

docs/how-to/make-per-application-backups.md

@@ -123,11 +123,7 @@ Once this include gets merged in, the resulting configuration would have a
 `keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
 
 When there's an option collision between the local file and the merged
-include, the local file's option takes precedence. And as of borgmatic 1.6.0,
-this feature performs a deep merge, meaning that values are merged at all
-levels in the two configuration files. Colliding list values are appended
-together. This allows you to include common configuration—up to full borgmatic
-configuration files—while overriding only the parts you want to customize.
+include, the local file's option takes precedence.
 
 Note that this `<<` include merging syntax is only for merging in mappings
 (configuration options and their values). But if you'd like to include a
@@ -139,6 +135,16 @@ global level, another `<<` within each configuration section, etc. (This is a
 YAML limitation.)
 
 
+### Deep merge
+
+<span class="minilink minilink-addedin">New in version 1.6.0</span> borgmatic
+performs a deep merge of merged include files, meaning that values are merged
+at all levels in the two configuration files. Colliding list values are
+appended together. This allows you to include common configuration—up to full
+borgmatic configuration files—while overriding only the parts you want to
+customize.
+
+
 ## Configuration overrides
 
 In more complex multi-application setups, you may want to override particular
@@ -203,3 +209,5 @@ indentation and a leading dash.)
 
 Be sure to quote your overrides if they contain spaces or other characters
 that your shell may interpret.
+
+An alternate to command-line overrides is passing in your values via [environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).

docs/how-to/monitor-your-backups.md

@@ -3,7 +3,7 @@ title: How to monitor your backups
 eleventyNavigation:
   key: 🚨 Monitor your backups
   parent: How-to guides
-  order: 5
+  order: 6
 ---
 
 ## Monitoring and alerting
@@ -61,8 +61,6 @@ one of them at most.
 You can use traditional monitoring software to consume borgmatic JSON output
 and track when the last successful backup occurred. See [scripting
 borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic)
-and [related
-software](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#related-software)
 below for how to configure this.
 
 ### Borg hosting providers
@@ -329,11 +327,6 @@ suppressed so as not to interfere with the captured JSON. Also note that JSON
 output only shows up at the console, and not in syslog.
 
 
-## Related software
-
- * [Borgmacator GNOME AppIndicator](https://github.com/N-Coder/borgmacator/)
-
-
 ### Latest backups
 
 All borgmatic actions that accept an "--archive" flag allow you to specify an

docs/how-to/provide-your-passwords.md

@@ -0,0 +1,82 @@
+---
+title: How to provide your passwords
+eleventyNavigation:
+  key: 🔒 Provide your passwords
+  parent: How-to guides
+  order: 2
+---
+## Environment variable interpolation
+
+If you want to use a Borg repository passphrase or database passwords with
+borgmatic, you can set them directly in your borgmatic configuration file,
+treating those secrets like any other option value. But if you'd rather store
+them outside of borgmatic, whether for convenience or security reasons, read
+on.
+
+<span class="minilink minilink-addedin">New in version 1.6.4</span> borgmatic
+supports interpolating arbitrary environment variables directly into option
+values in your configuration file. That means you can instruct borgmatic to
+pull your repository passphrase, your database passwords, or any other option
+values from environment variables. For instance:
+
+```yaml
+storage:
+    encryption_passphrase: ${MY_PASSPHRASE}
+```
+
+This uses the `MY_PASSPHRASE` environment variable as your encryption
+passphrase. Note that the `{` `}` brackets are required. Just `$MY_PASSPHRASE`
+will not work.
+
+In the case of `encryption_passphrase` in particular, an alternate approach
+is to use Borg's `BORG_PASSPHRASE` environment variable, which doesn't even
+require setting an explicit `encryption_passphrase` value in borgmatic's
+configuration file.
+
+For [database
+configuration](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/),
+the same approach applies. For example:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: users
+          password: ${MY_DATABASE_PASSWORD}
+```
+
+This uses the `MY_DATABASE_PASSWORD` environment variable as your database
+password.
+
+
+### Interpolation defaults
+
+If you'd like to set a default for your environment variables, you can do so with the following syntax:
+
+```yaml
+storage:
+    encryption_passphrase: ${MY_PASSPHRASE:-defaultpass}
+```
+
+Here, "`defaultpass`" is the default passphrase if the `MY_PASSPHRASE`
+environment variable is not set. Without a default, if the environment
+variable doesn't exist, borgmatic will error.
+
+
+### Disabling interpolation
+
+To disable this environment variable interpolation feature entirely, you can
+pass the `--no-environment-interpolation` flag on the command-line.
+
+
+### Related features
+
+Another way to override particular options within a borgmatic configuration
+file is to use a [configuration
+override](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides)
+on the command-line. But please be aware of the security implications of
+specifying secrets on the command-line.
+
+Additionally, borgmatic action hooks support their own [variable
+interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation),
+although in that case it's for particular borgmatic runtime values rather than
+(only) environment variables.

docs/how-to/run-arbitrary-borg-commands.md

@@ -3,7 +3,7 @@ title: How to run arbitrary Borg commands
 eleventyNavigation:
   key: 🔧 Run arbitrary Borg commands
   parent: How-to guides
-  order: 10
+  order: 11
 ---
 ## Running Borg with borgmatic
 

docs/how-to/upgrade.md

@@ -3,7 +3,7 @@ title: How to upgrade borgmatic
 eleventyNavigation:
   key: 📦 Upgrade borgmatic
   parent: How-to guides
-  order: 11
+  order: 12
 ---
 ## Upgrading
 

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.6.3'
+VERSION = '1.6.4.dev0'
 
 
 setup(

tests/unit/config/test_environment.py

@@ -0,0 +1,85 @@
+import pytest
+
+from borgmatic.config import environment as module
+
+
+def test_env(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    config = {'key': 'Hello $MY_CUSTOM_VALUE'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello $MY_CUSTOM_VALUE'}
+
+
+def test_env_braces(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello foo'}
+
+def test_env_multi(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    monkeypatch.setenv('MY_CUSTOM_VALUE2', 'bar')
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE}${MY_CUSTOM_VALUE2}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello foobar'}
+
+def test_env_escape(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    monkeypatch.setenv('MY_CUSTOM_VALUE2', 'bar')
+    config = {'key': r'Hello ${MY_CUSTOM_VALUE} \${MY_CUSTOM_VALUE}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': r'Hello foo ${MY_CUSTOM_VALUE}'}
+
+
+def test_env_default_value(monkeypatch):
+    monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False)
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello bar'}
+
+
+def test_env_unknown(monkeypatch):
+    monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False)
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
+    with pytest.raises(ValueError):
+        module.resolve_env_variables(config)
+
+
+def test_env_full(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    monkeypatch.delenv('MY_CUSTOM_VALUE2', raising=False)
+    config = {
+        'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
+        'dict': {
+            'key': 'value',
+            'anotherdict': {
+                'key': 'My ${MY_CUSTOM_VALUE} here',
+                'other': '${MY_CUSTOM_VALUE}',
+                'escaped': r'\${MY_CUSTOM_VALUE}',
+                'list': [
+                    '/home/${MY_CUSTOM_VALUE}/.local',
+                    '/var/log/',
+                    '/home/${MY_CUSTOM_VALUE2:-bar}/.config',
+                ],
+            },
+        },
+        'list': [
+            '/home/${MY_CUSTOM_VALUE}/.local',
+            '/var/log/',
+            '/home/${MY_CUSTOM_VALUE2-bar}/.config',
+        ],
+    }
+    module.resolve_env_variables(config)
+    assert config == {
+        'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
+        'dict': {
+            'key': 'value',
+            'anotherdict': {
+                'key': 'My foo here',
+                'other': 'foo',
+                'escaped': '${MY_CUSTOM_VALUE}',
+                'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'],
+            },
+        },
+        'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'],
+    }