Revamped the credentials to load them much closer to where they're used (#966).

borgmatic/borg/environment.py

@@ -1,6 +1,7 @@
 import os
 
 import borgmatic.borg.passcommand
+import borgmatic.hooks.credential.tag
 
 OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_base_directory': 'BORG_BASE_DIR',
@@ -14,6 +15,8 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'temporary_directory': 'TMPDIR',
 }
 
+CREDENTIAL_OPTIONS = {'encryption_passphrase'}
+
 DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = {
     'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
     'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
@@ -37,7 +40,10 @@ def make_environment(config):
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
         value = config.get(option_name)
 
-        if value:
+        if option_name in CREDENTIAL_OPTIONS:
+            value = borgmatic.hooks.credential.tag.resolve_credential(value) 
+
+        if value is not None:
             environment[environment_variable_name] = str(value)
 
     passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config)

borgmatic/commands/borgmatic.py

@@ -534,15 +534,15 @@ def run_actions(
 
 
 def load_configurations(
-    config_filenames, overrides=None, resolve_env=True, resolve_credentials=True
+    config_filenames, overrides=None, resolve_env=True
 ):
     '''
     Given a sequence of configuration filenames, a sequence of configuration file override strings
-    in the form of "option.suboption=value", whether to resolve environment variables, and whether
-    to resolve credentials, load and validate each configuration file. Return the results as a tuple
-    of: dict of configuration filename to corresponding parsed configuration, a sequence of paths
-    for all loaded configuration files (including includes), and a sequence of logging.LogRecord
-    instances containing any parse errors.
+    in the form of "option.suboption=value", and whether to resolve environment variables, load and
+    validate each configuration file. Return the results as a tuple of: dict of configuration
+    filename to corresponding parsed configuration, a sequence of paths for all loaded configuration
+    files (including includes), and a sequence of logging.LogRecord instances containing any parse
+    errors.
 
     Log records are returned here instead of being logged directly because logging isn't yet
     initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this
@@ -572,7 +572,6 @@ def load_configurations(
                 validate.schema_filename(),
                 overrides,
                 resolve_env,
-                resolve_credentials,
             )
             config_paths.update(paths)
             logs.extend(parse_logs)
@@ -922,7 +921,6 @@ def main(extra_summary_logs=[]):  # pragma: no cover
         config_filenames,
         global_arguments.overrides,
         resolve_env=global_arguments.resolve_env and not validate,
-        resolve_credentials=not validate,
     )
     configuration_parse_errors = (
         (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False

borgmatic/config/credential.py

@@ -1,37 +0,0 @@
-import borgmatic.hooks.dispatch
-
-UNSPECIFIED = object()
-
-
-def resolve_credentials(config, item=UNSPECIFIED):
-    '''
-    Resolves values like "!credential hookname credentialname" from the given configuration by
-    calling relevant hooks to get the actual credential values. The item parameter is used to
-    support recursing through the config hierarchy; it represents the current piece of config being
-    looked at.
-
-    Raise ValueError if the config could not be parsed or the credential could not be loaded.
-    '''
-    if item is UNSPECIFIED:
-        item = config
-
-    if isinstance(item, str):
-        if item.startswith('!credential '):
-            try:
-                (tag_name, hook_name, credential_name) = item.split(' ', 2)
-            except ValueError:
-                raise ValueError(f'Cannot load credential with invalid syntax "{item}"')
-
-            return borgmatic.hooks.dispatch.call_hook(
-                'load_credential', config, hook_name, credential_name
-            )
-
-    if isinstance(item, list):
-        for index, subitem in enumerate(item):
-            item[index] = resolve_credentials(config, subitem)
-
-    if isinstance(item, dict):
-        for key, value in item.items():
-            item[key] = resolve_credentials(config, value)
-
-    return item

borgmatic/config/validate.py

@@ -5,7 +5,6 @@ import jsonschema
 import ruamel.yaml
 
 import borgmatic.config
-import borgmatic.config.credential
 from borgmatic.config import constants, environment, load, normalize, override
 
 
@@ -86,14 +85,14 @@ def apply_logical_validation(config_filename, parsed_configuration):
 
 
 def parse_configuration(
-    config_filename, schema_filename, overrides=None, resolve_env=True, resolve_credentials=True
+    config_filename, schema_filename, overrides=None, resolve_env=True
 ):
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
     rendition of JSON Schema format, a sequence of configuration file override strings in the form
-    of "option.suboption=value", whether to resolve environment variables, and whether to resolve
-    credentials, return the parsed configuration as a data structure of nested dicts and lists
-    corresponding to the schema. Example return value:
+    of "option.suboption=value", and whether to resolve environment variables, return the parsed
+    configuration as a data structure of nested dicts and lists corresponding to the schema. Example
+    return value:
 
         {
             'source_directories': ['/home', '/etc'],
@@ -122,9 +121,6 @@ def parse_configuration(
     if resolve_env:
         environment.resolve_env_variables(config)
 
-    if resolve_credentials:
-        borgmatic.config.credential.resolve_credentials(config)
-
     logs = normalize.normalize(config_filename, config)
 
     try:

borgmatic/hooks/credential/systemd.py

@@ -30,6 +30,6 @@ def load_credential(hook_config, config, credential_name):
         with open(os.path.join(credentials_directory, credential_name)) as credential_file:
             return credential_file.read().rstrip(os.linesep)
     except (FileNotFoundError, OSError) as error:
-        logger.error(error)
+        logger.warning(error)
 
         raise ValueError(f'Cannot load credential "{credential_name}" from file: {error.filename}')

borgmatic/hooks/credential/tag.py

@@ -0,0 +1,29 @@
+import functools
+
+import borgmatic.hooks.dispatch
+
+IS_A_HOOK = False
+
+
+@functools.cache
+def resolve_credential(tag):
+    '''
+    Given a configuration tag string like "!credential hookname credentialname", resolve it by
+    calling the relevant hook to get the actual credential value. If the given tag is not actually a
+    credential tag, then return the value unchanged.
+
+    Cache the value so repeated calls to this function don't need to load the credential repeatedly.
+
+    Raise ValueError if the config could not be parsed or the credential could not be loaded.
+    '''
+    if tag and tag.startswith('!credential '):
+        try:
+            (tag_name, hook_name, credential_name) = tag.split(' ', 2)
+        except ValueError:
+            raise ValueError(f'Cannot load credential with invalid syntax "{tag}"')
+
+        return borgmatic.hooks.dispatch.call_hook(
+            'load_credential', {}, hook_name, credential_name
+        )
+
+    return tag

borgmatic/hooks/monitoring/ntfy.py

@@ -2,6 +2,8 @@ import logging
 
 import requests
 
+import borgmatic.hooks.credential.tag
+
 logger = logging.getLogger(__name__)
 
 
@@ -47,9 +49,14 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
             'X-Tags': state_config.get('tags'),
         }
 
-        username = hook_config.get('username')
-        password = hook_config.get('password')
-        access_token = hook_config.get('access_token')
+        try:
+            username = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('username'))
+            password = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('password'))
+            access_token = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('access_token'))
+        except ValueError as error:
+            logger.warning(f'Ntfy credential error: {error}')
+            return
+
         auth = None
 
         if access_token is not None:

borgmatic/hooks/monitoring/pagerduty.py

@@ -5,6 +5,7 @@ import platform
 
 import requests
 
+import borgmatic.hooks.credential.tag
 from borgmatic.hooks.monitoring import monitor
 
 logger = logging.getLogger(__name__)
@@ -39,11 +40,17 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     if dry_run:
         return
 
+    try:
+        inegration_key = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('integration_key'))
+    except ValueError as error:
+        logger.warning(f'PagerDuty credential error: {error}')
+        return
+
     hostname = platform.node()
     local_timestamp = datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat()
     payload = json.dumps(
         {
-            'routing_key': hook_config['integration_key'],
+            'routing_key': integration_key,
             'event_action': 'trigger',
             'payload': {
                 'summary': f'backup failed on {hostname}',

borgmatic/hooks/monitoring/pushover.py

@@ -2,6 +2,8 @@ import logging
 
 import requests
 
+import borgmatic.hooks.credential.tag
+
 logger = logging.getLogger(__name__)
 
 
@@ -32,8 +34,12 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
 
     state_config = hook_config.get(state.name.lower(), {})
 
-    token = hook_config.get('token')
-    user = hook_config.get('user')
+    try:
+        token = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('token'))
+        user = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('user'))
+    except ValueError as error:
+        logger.warning(f'Pushover credential error: {error}')
+        return
 
     logger.info(f'Updating Pushover{dry_run_label}')
 

borgmatic/hooks/monitoring/zabbix.py

@@ -2,6 +2,8 @@ import logging
 
 import requests
 
+import borgmatic.hooks.credential.tag
+
 logger = logging.getLogger(__name__)
 
 
@@ -34,10 +36,15 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
         },
     )
 
+    try:
+        username = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('username'))
+        password = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('password'))
+        api_key = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('api_key'))
+    except ValueError as error:
+        logger.warning(f'Zabbix credential error: {error}')
+        return
+
     server = hook_config.get('server')
-    username = hook_config.get('username')
-    password = hook_config.get('password')
-    api_key = hook_config.get('api_key')
     itemid = hook_config.get('itemid')
     host = hook_config.get('host')
     key = hook_config.get('key')