Add a "!credential" tag for loading systemd credentials into borgmatic configuration (#966).

NEWS

@@ -1,4 +1,7 @@
 1.9.10.dev0
+ * #966: Add a "!credential" tag for loading systemd credentials into borgmatic configuration
+   files. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
  * #987: Fix a "list" action error when the "encryption_passcommand" option is set.
  * #987: When both "encryption_passcommand" and "encryption_passphrase" are configured, prefer
    "encryption_passphrase" even if it's an empty value.

borgmatic/commands/borgmatic.py

@@ -533,15 +533,20 @@ def run_actions(
     )
 
 
-def load_configurations(config_filenames, overrides=None, resolve_env=True):
+def load_configurations(
+    config_filenames, overrides=None, resolve_env=True, resolve_credentials=True
+):
     '''
-    Given a sequence of configuration filenames, load and validate each configuration file. Return
-    the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
-    a sequence of paths for all loaded configuration files (including includes), and a sequence of
-    logging.LogRecord instances containing any parse errors.
+    Given a sequence of configuration filenames, a sequence of configuration file override strings
+    in the form of "option.suboption=value", whether to resolve environment variables, and whether
+    to resolve credentials, load and validate each configuration file. Return the results as a tuple
+    of: dict of configuration filename to corresponding parsed configuration, a sequence of paths
+    for all loaded configuration files (including includes), and a sequence of logging.LogRecord
+    instances containing any parse errors.
 
     Log records are returned here instead of being logged directly because logging isn't yet
-    initialized at this point!
+    initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this
+    approach could change.)
     '''
     # Dict mapping from config filename to corresponding parsed config dict.
     configs = collections.OrderedDict()
@@ -563,7 +568,11 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
         )
         try:
             configs[config_filename], paths, parse_logs = validate.parse_configuration(
-                config_filename, validate.schema_filename(), overrides, resolve_env
+                config_filename,
+                validate.schema_filename(),
+                overrides,
+                resolve_env,
+                resolve_credentials,
             )
             config_paths.update(paths)
             logs.extend(parse_logs)
@@ -907,9 +916,13 @@ def main(extra_summary_logs=[]):  # pragma: no cover
         print(borgmatic.commands.completion.fish.fish_completion())
         sys.exit(0)
 
+    validate = bool('validate' in arguments)
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
     configs, config_paths, parse_logs = load_configurations(
-        config_filenames, global_arguments.overrides, global_arguments.resolve_env
+        config_filenames,
+        global_arguments.overrides,
+        resolve_env=global_arguments.resolve_env and not validate,
+        resolve_credentials=not validate,
     )
     configuration_parse_errors = (
         (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False

borgmatic/config/credential.py

@@ -0,0 +1,33 @@
+import borgmatic.hooks.dispatch
+
+
+def resolve_credentials(config, item=None):
+    '''
+    Resolves values like "!credential hookname credentialname" from the given configuration by
+    calling relevant hooks to get the actual credential values.
+
+    Raise ValueError if the config could not be parsed or the credential could not be loaded.
+    '''
+    if not item:
+        item = config
+
+    if isinstance(item, str):
+        if item.startswith('!credential '):
+            try:
+                (tag_name, hook_name, credential_name) = item.split(' ', 2)
+            except ValueError:
+                raise ValueError(f'Cannot load credential with invalid syntax "{item}"')
+
+            return borgmatic.hooks.dispatch.call_hook(
+                'load_credential', config, hook_name, credential_name
+            )
+
+    if isinstance(item, list):
+        for index, subitem in enumerate(item):
+            item[index] = resolve_credentials(config, subitem)
+
+    if isinstance(item, dict):
+        for key, value in item.items():
+            item[key] = resolve_credentials(config, value)
+
+    return item

borgmatic/config/load.py

@@ -69,7 +69,7 @@ def include_configuration(loader, filename_node, include_directory, config_paths
         ]
 
     raise ValueError(
-        '!include value is not supported; use a single filename or a list of filenames'
+        'The value given for the !include tag is not supported; use a single filename or a list of filenames instead'
     )
 
 
@@ -104,6 +104,23 @@ def raise_omit_node_error(loader, node):
     )
 
 
+def reserialize_tag_node(loader, tag_node):
+    '''
+    Given a ruamel.yaml loader and a node for a tag and value, convert the node back into a string
+    of the form "!tagname value" and return it. The idea is that downstream code, rather than this
+    file's YAML loading logic, should be responsible for interpreting this particular tag—since the
+    downstream code actually understands the meaning behind the tag.
+
+    Raise ValueError if the tag node's value isn't a string.
+    '''
+    if isinstance(tag_node.value, str):
+        return f'{tag_node.tag} {tag_node.value}'
+
+    raise ValueError(
+        f'The value given for the {tag_node.tag} tag is not supported; use a string instead'
+    )
+
+
 class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
     A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
@@ -122,6 +139,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
                 config_paths=config_paths,
             ),
         )
+        self.add_constructor('!credential', reserialize_tag_node)
 
         # These are catch-all error handlers for tags that don't get applied and removed by
         # deep_merge_nodes() below.

borgmatic/config/validate.py

@@ -5,6 +5,7 @@ import jsonschema
 import ruamel.yaml
 
 import borgmatic.config
+import borgmatic.config.credential
 from borgmatic.config import constants, environment, load, normalize, override
 
 
@@ -84,12 +85,15 @@ def apply_logical_validation(config_filename, parsed_configuration):
             )
 
 
-def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True):
+def parse_configuration(
+    config_filename, schema_filename, overrides=None, resolve_env=True, resolve_credentials=True
+):
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
     rendition of JSON Schema format, a sequence of configuration file override strings in the form
-    of "option.suboption=value", return the parsed configuration as a data structure of nested dicts
-    and lists corresponding to the schema. Example return value:
+    of "option.suboption=value", whether to resolve environment variables, and whether to resolve
+    credentials, return the parsed configuration as a data structure of nested dicts and lists
+    corresponding to the schema. Example return value:
 
         {
             'source_directories': ['/home', '/etc'],
@@ -118,12 +122,16 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
     if resolve_env:
         environment.resolve_env_variables(config)
 
+    if resolve_credentials:
+        borgmatic.config.credential.resolve_credentials(config)
+
     logs = normalize.normalize(config_filename, config)
 
     try:
         validator = jsonschema.Draft7Validator(schema)
     except AttributeError:  # pragma: no cover
         validator = jsonschema.Draft4Validator(schema)
+
     validation_errors = tuple(validator.iter_errors(config))
 
     if validation_errors:

borgmatic/hooks/credential/systemd.py

@@ -0,0 +1,39 @@
+import functools
+import logging
+import os
+import re
+
+import borgmatic.config.paths
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+CREDENTIAL_NAME_PATTERN = re.compile(r'^\w+$')
+
+
+def load_credential(hook_config, config, credential_name):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential name to load, read
+    the credential from the corresonding systemd credential file and return it.
+
+    Raise ValueError if the systemd CREDENTIALS_DIRECTORY environment variable is not set, the
+    credential name is invalid, or the credential file cannot be read.
+    '''
+    credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
+
+    if not credentials_directory:
+        raise ValueError(
+            f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set'
+        )
+
+    if not CREDENTIAL_NAME_PATTERN.match(credential_name):
+        raise ValueError(f'Cannot load invalid credential name "{credential_name}"')
+
+    try:
+        with open(os.path.join(credentials_directory, credential_name)) as credential_file:
+            return credential_file.read().rstrip(os.linesep)
+    except (FileNotFoundError, OSError) as error:
+        logger.error(error)
+
+        raise ValueError(f'Cannot load credential "{credential_name}" from file: {error.filename}')