add support for basic auth on self-hosted instances

Reviewed-on: borgmatic-collective/borgmatic#549

NEWS

@@ -1,8 +1,25 @@
-1.6.3.dev0
+1.6.5
+ * #553: Fix logging to include the full traceback when Borg experiences an internal error, not just
+   the first few lines.
+ * #554: Fix all monitoring hooks to warn if the server returns an HTTP 4xx error. This can happen
+   with Healthchecks, for instance, when using an invalid ping URL.
+ * #555: Fix environment variable plumbing so options like "encryption_passphrase" and
+   "encryption_passcommand" in one configuration file aren't used for other configuration files.
+
+1.6.4
+ * #546, #382: Keep your repository passphrases and database passwords outside of borgmatic's
+   configuration file with environment variable interpolation. See the documentation for more
+   information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+
+1.6.3
  * #541: Add "borgmatic list --find" flag for searching for files across multiple archives, useful
    for hunting down that file you accidentally deleted so you can extract it. See the documentation
    for more information:
    https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#searching-for-a-file
+ * #543: Add a monitoring hook for sending push notifications via ntfy. See the documentation for
+   more information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook
+ * Fix Bash completion script to no longer alter your shell's settings (complain about unset
+   variables or error on pipe failures).
  * Deprecate "borgmatic list --successful" flag, as listing only non-checkpoint (successful)
    archives is now the default in newer versions of Borg.
 

README.md

@@ -71,6 +71,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 <a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 
 

borgmatic/borg/borg.py

@@ -1,5 +1,6 @@
 import logging
 
+from borgmatic.borg import environment
 from borgmatic.borg.flags import make_flags
 from borgmatic.execute import execute_command
 
@@ -51,5 +52,8 @@ def run_arbitrary_borg(
     )
 
     return execute_command(
-        full_command, output_log_level=logging.WARNING, borg_local_path=local_path,
+        full_command,
+        output_log_level=logging.WARNING,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
     )

borgmatic/borg/check.py

@@ -5,7 +5,7 @@ import logging
 import os
 import pathlib
 
-from borgmatic.borg import extract, info, state
+from borgmatic.borg import environment, extract, info, state
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 DEFAULT_CHECKS = (
@@ -304,16 +304,22 @@ def check_archives(
             + (repository,)
         )
 
+        borg_environment = environment.make_environment(storage_config)
+
         # The Borg repair option triggers an interactive prompt, which won't work when output is
         # captured. And progress messes with the terminal directly.
         if repair or progress:
-            execute_command(full_command, output_file=DO_NOT_CAPTURE)
+            execute_command(
+                full_command, output_file=DO_NOT_CAPTURE, extra_environment=borg_environment
+            )
         else:
-            execute_command(full_command)
+            execute_command(full_command, extra_environment=borg_environment)
 
         for check in checks:
             write_check_time(make_check_time_path(location_config, borg_repository_id, check))
 
     if 'extract' in checks:
-        extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)
+        extract.extract_last_archive_dry_run(
+            storage_config, repository, lock_wait, local_path, remote_path
+        )
         write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract'))

borgmatic/borg/compact.py

@@ -1,5 +1,6 @@
 import logging
 
+from borgmatic.borg import environment
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
@@ -38,4 +39,9 @@ def compact_segments(
     )
 
     if not dry_run:
-        execute_command(full_command, output_log_level=logging.INFO, borg_local_path=local_path)
+        execute_command(
+            full_command,
+            output_log_level=logging.INFO,
+            borg_local_path=local_path,
+            extra_environment=environment.make_environment(storage_config),
+        )

borgmatic/borg/create.py

@@ -5,7 +5,7 @@ import os
 import pathlib
 import tempfile
 
-from borgmatic.borg import feature, state
+from borgmatic.borg import environment, feature, state
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
 
 logger = logging.getLogger(__name__)
@@ -317,6 +317,8 @@ def create_archive(
     # the terminal directly.
     output_file = DO_NOT_CAPTURE if progress else None
 
+    borg_environment = environment.make_environment(storage_config)
+
     if stream_processes:
         return execute_command_with_processes(
             full_command,
@@ -325,6 +327,7 @@ def create_archive(
             output_file,
             borg_local_path=local_path,
             working_directory=working_directory,
+            extra_environment=borg_environment,
         )
 
     return execute_command(
@@ -333,4 +336,5 @@ def create_archive(
         output_file,
         borg_local_path=local_path,
         working_directory=working_directory,
+        extra_environment=borg_environment,
     )

borgmatic/borg/environment.py

@@ -1,5 +1,3 @@
-import os
-
 OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_base_directory': 'BORG_BASE_DIR',
     'borg_config_directory': 'BORG_CONFIG_DIR',
@@ -18,21 +16,24 @@ DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
 }
 
 
-def initialize(storage_config):
-    for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
+def make_environment(storage_config):
+    '''
+    Given a borgmatic storage configuration dict, return its options converted to a Borg environment
+    variable dict.
+    '''
+    environment = {}
 
-        # Options from borgmatic configuration take precedence over already set BORG_* environment
-        # variables.
-        value = storage_config.get(option_name) or os.environ.get(environment_variable_name)
+    for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name)
 
         if value:
-            os.environ[environment_variable_name] = value
-        else:
-            os.environ.pop(environment_variable_name, None)
+            environment[environment_variable_name] = value
 
     for (
         option_name,
         environment_variable_name,
     ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
         value = storage_config.get(option_name, False)
-        os.environ[environment_variable_name] = 'yes' if value else 'no'
+        environment[environment_variable_name] = 'yes' if value else 'no'
+
+    return environment

borgmatic/borg/export_tar.py

@@ -1,6 +1,7 @@
 import logging
 import os
 
+from borgmatic.borg import environment
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
@@ -61,4 +62,5 @@ def export_tar_archive(
         output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
         output_log_level=output_log_level,
         borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
     )

borgmatic/borg/extract.py

@@ -2,13 +2,15 @@ import logging
 import os
 import subprocess
 
-from borgmatic.borg import feature
+from borgmatic.borg import environment, feature
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg', remote_path=None):
+def extract_last_archive_dry_run(
+    storage_config, repository, lock_wait=None, local_path='borg', remote_path=None
+):
     '''
     Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
     dry-run.
@@ -29,8 +31,13 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         + (repository,)
     )
 
+    borg_environment = environment.make_environment(storage_config)
+
     list_output = execute_command(
-        full_list_command, output_log_level=None, borg_local_path=local_path
+        full_list_command,
+        output_log_level=None,
+        borg_local_path=local_path,
+        extra_environment=borg_environment,
     )
 
     try:
@@ -52,7 +59,9 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         )
     )
 
-    execute_command(full_extract_command, working_directory=None)
+    execute_command(
+        full_extract_command, working_directory=None, extra_environment=borg_environment
+    )
 
 
 def extract_archive(
@@ -106,11 +115,16 @@ def extract_archive(
         + (tuple(paths) if paths else ())
     )
 
+    borg_environment = environment.make_environment(storage_config)
+
     # The progress output isn't compatible with captured and logged output, as progress messes with
     # the terminal directly.
     if progress:
         return execute_command(
-            full_command, output_file=DO_NOT_CAPTURE, working_directory=destination_path
+            full_command,
+            output_file=DO_NOT_CAPTURE,
+            working_directory=destination_path,
+            extra_environment=borg_environment,
         )
         return None
 
@@ -120,8 +134,11 @@ def extract_archive(
             output_file=subprocess.PIPE,
             working_directory=destination_path,
             run_to_completion=False,
+            extra_environment=borg_environment,
         )
 
     # Don't give Borg local path, so as to error on warnings, as Borg only gives a warning if the
     # restore paths don't exist in the archive!
-    execute_command(full_command, working_directory=destination_path)
+    execute_command(
+        full_command, working_directory=destination_path, extra_environment=borg_environment
+    )

borgmatic/borg/info.py

@@ -1,5 +1,6 @@
 import logging
 
+from borgmatic.borg import environment
 from borgmatic.borg.flags import make_flags, make_flags_from_arguments
 from borgmatic.execute import execute_command
 
@@ -42,4 +43,5 @@ def display_archives_info(
         full_command,
         output_log_level=None if info_arguments.json else logging.WARNING,
         borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
     )

borgmatic/borg/init.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 import subprocess
 
-from borgmatic.borg import info
+from borgmatic.borg import environment, info
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
@@ -54,4 +54,9 @@ def initialize_repository(
     )
 
     # Do not capture output here, so as to support interactive prompts.
-    execute_command(init_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)
+    execute_command(
+        init_command,
+        output_file=DO_NOT_CAPTURE,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
+    )

borgmatic/borg/list.py

@@ -2,6 +2,7 @@ import copy
 import logging
 import re
 
+from borgmatic.borg import environment
 from borgmatic.borg.flags import make_flags, make_flags_from_arguments
 from borgmatic.execute import execute_command
 
@@ -31,7 +32,12 @@ def resolve_archive_name(repository, archive, storage_config, local_path='borg',
         + ('--short', repository)
     )
 
-    output = execute_command(full_command, output_log_level=None, borg_local_path=local_path)
+    output = execute_command(
+        full_command,
+        output_log_level=None,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
+    )
     try:
         latest_archive = output.strip().splitlines()[-1]
     except IndexError:
@@ -111,6 +117,8 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
     archive. Or, if list_arguments.find_paths are given, list the files by searching across multiple
     archives.
     '''
+    borg_environment = environment.make_environment(storage_config)
+
     # If there are any paths to find (and there's not a single archive already selected), start by
     # getting a list of archives to search.
     if list_arguments.find_paths and not list_arguments.archive:
@@ -127,6 +135,7 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
                 ),
                 output_log_level=None,
                 borg_local_path=local_path,
+                extra_environment=borg_environment,
             )
             .strip('\n')
             .split('\n')
@@ -154,6 +163,7 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
             main_command,
             output_log_level=None if list_arguments.json else logging.WARNING,
             borg_local_path=local_path,
+            extra_environment=borg_environment,
         )
 
         if list_arguments.json:

borgmatic/borg/mount.py

@@ -1,5 +1,6 @@
 import logging
 
+from borgmatic.borg import environment
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
@@ -38,9 +39,16 @@ def mount_archive(
         + (tuple(paths) if paths else ())
     )
 
+    borg_environment = environment.make_environment(storage_config)
+
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
     if foreground:
-        execute_command(full_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)
+        execute_command(
+            full_command,
+            output_file=DO_NOT_CAPTURE,
+            borg_local_path=local_path,
+            extra_environment=borg_environment,
+        )
         return
 
-    execute_command(full_command, borg_local_path=local_path)
+    execute_command(full_command, borg_local_path=local_path, extra_environment=borg_environment)

borgmatic/borg/prune.py

@@ -1,5 +1,6 @@
 import logging
 
+from borgmatic.borg import environment
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
@@ -72,4 +73,9 @@ def prune_archives(
     else:
         output_log_level = logging.INFO
 
-    execute_command(full_command, output_log_level=output_log_level, borg_local_path=local_path)
+    execute_command(
+        full_command,
+        output_log_level=output_log_level,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
+    )

borgmatic/borg/version.py

@@ -1,13 +1,14 @@
 import logging
 
+from borgmatic.borg import environment
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def local_borg_version(local_path='borg'):
+def local_borg_version(storage_config, local_path='borg'):
     '''
-    Given a local Borg binary path, return a version string for it.
+    Given a storage configuration dict and a local Borg binary path, return a version string for it.
 
     Raise OSError or CalledProcessError if there is a problem running Borg.
     Raise ValueError if the version cannot be parsed.
@@ -17,7 +18,12 @@ def local_borg_version(local_path='borg'):
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
     )
-    output = execute_command(full_command, output_log_level=None, borg_local_path=local_path)
+    output = execute_command(
+        full_command,
+        output_log_level=None,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
+    )
 
     try:
         return output.split(' ')[1].strip()

borgmatic/commands/arguments.py

@@ -188,6 +188,12 @@ def make_parsers():
         action='extend',
         help='One or more configuration file options to override with specified values',
     )
+    global_group.add_argument(
+        '--no-environment-interpolation',
+        dest='resolve_env',
+        action='store_false',
+        help='Do not resolve environment variables in configuration file',
+    )
     global_group.add_argument(
         '--bash-completion',
         default=False,

borgmatic/commands/borgmatic.py

@@ -16,7 +16,6 @@ from borgmatic.borg import borg as borg_borg
 from borgmatic.borg import check as borg_check
 from borgmatic.borg import compact as borg_compact
 from borgmatic.borg import create as borg_create
-from borgmatic.borg import environment as borg_environment
 from borgmatic.borg import export_tar as borg_export_tar
 from borgmatic.borg import extract as borg_extract
 from borgmatic.borg import feature as borg_feature
@@ -60,14 +59,13 @@ def run_configuration(config_filename, config, arguments):
     remote_path = location.get('remote_path')
     retries = storage.get('retries', 0)
     retry_wait = storage.get('retry_wait', 0)
-    borg_environment.initialize(storage)
     encountered_error = None
     error_repository = ''
     using_primary_action = {'prune', 'compact', 'create', 'check'}.intersection(arguments)
     monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity)
 
     try:
-        local_borg_version = borg_version.local_borg_version(local_path)
+        local_borg_version = borg_version.local_borg_version(storage, local_path)
     except (OSError, CalledProcessError, ValueError) as error:
         yield from log_error_records(
             '{}: Error getting local Borg version'.format(config_filename), error
@@ -650,7 +648,7 @@ def run_actions(
             )
 
 
-def load_configurations(config_filenames, overrides=None):
+def load_configurations(config_filenames, overrides=None, resolve_env=True):
     '''
     Given a sequence of configuration filenames, load and validate each configuration file. Return
     the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
@@ -664,7 +662,7 @@ def load_configurations(config_filenames, overrides=None):
     for config_filename in config_filenames:
         try:
             configs[config_filename] = validate.parse_configuration(
-                config_filename, validate.schema_filename(), overrides
+                config_filename, validate.schema_filename(), overrides, resolve_env
             )
         except PermissionError:
             logs.extend(
@@ -835,7 +833,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
         logger.info('Unmounting mount point {}'.format(arguments['umount'].mount_point))
         try:
             borg_umount.unmount_archive(
-                mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs)
+                mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs),
             )
         except (CalledProcessError, OSError) as error:
             yield from log_error_records('Error unmounting mount point', error)
@@ -892,7 +890,9 @@ def main():  # pragma: no cover
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
-    configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
+    configs, parse_logs = load_configurations(
+        config_filenames, global_arguments.overrides, global_arguments.resolve_env
+    )
 
     any_json_flags = any(
         getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()

borgmatic/commands/completion.py

@@ -30,7 +30,6 @@ def bash_completion():
     # Avert your eyes.
     return '\n'.join(
         (
-            'set -uo pipefail',
             'check_version() {',
             '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
             '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',

borgmatic/config/environment.py

@@ -0,0 +1,42 @@
+import os
+import re
+
+_VARIABLE_PATTERN = re.compile(
+    r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})'
+)
+
+
+def _resolve_string(matcher):
+    '''
+    Get the value from environment given a matcher containing a name and an optional default value.
+    If the variable is not defined in environment and no default value is provided, an Error is raised.
+    '''
+    if matcher.group('escape') is not None:
+        # in case of escaped envvar, unescape it
+        return matcher.group('variable')
+    # resolve the env var
+    name, default = matcher.group('name'), matcher.group('default')
+    out = os.getenv(name, default=default)
+    if out is None:
+        raise ValueError('Cannot find variable ${name} in environment'.format(name=name))
+    return out
+
+
+def resolve_env_variables(item):
+    '''
+    Resolves variables like or ${FOO} from given configuration with values from process environment
+    Supported formats:
+     - ${FOO} will return FOO env variable
+     - ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
+
+    If any variable is missing in environment and no default value is provided, an Error is raised.
+    '''
+    if isinstance(item, str):
+        return _VARIABLE_PATTERN.sub(_resolve_string, item)
+    if isinstance(item, list):
+        for i, subitem in enumerate(item):
+            item[i] = resolve_env_variables(subitem)
+    if isinstance(item, dict):
+        for key, value in item.items():
+            item[key] = resolve_env_variables(value)
+    return item

borgmatic/config/schema.yaml

@@ -900,6 +900,119 @@ properties:
                     https://docs.mongodb.com/database-tools/mongodump/ and
                     https://docs.mongodb.com/database-tools/mongorestore/ for
                     details.
+            ntfy:
+                type: object
+                required: ['topic']
+                additionalProperties: false
+                properties:
+                    topic:
+                        type: string
+                        description: |
+                            The topic to publish to.
+                            (https://ntfy.sh/docs/publish/)
+                        example: topic
+                    server:
+                        type: string
+                        description: |
+                            The address of your self-hosted ntfy.sh instance.
+                        example: https://ntfy.your-domain.com
+                    username:
+                        type: string
+                        description: |
+                            The username for your self-hosted ntfy.sh instance
+                            (ignored for the upstream-hosted instance)
+                        example: borgmatic
+                    password:
+                        type: string
+                        description: |
+                            The password for your self-hosted ntfy.sh instance.
+                            (ignored for the upstream-hosted instance)
+                        example: superSecretP@ssw0rd
+                    start:
+                        type: object
+                        properties:
+                            title:
+                                type: string
+                                description: |
+                                    The title of the message
+                                example: Ping!
+                            message:
+                                type: string
+                                description: |
+                                    The message body to publish.
+                                example: Your backups have failed.
+                            priority:
+                                type: string
+                                description: |
+                                    The priority to set.
+                                example: urgent
+                            tags:
+                                type: string
+                                description: |
+                                    Tags to attach to the message.
+                                example: incoming_envelope
+                    finish:
+                        type: object
+                        properties:
+                            title:
+                                type: string
+                                description: |
+                                    The title of the message.
+                                example: Ping!
+                            message:
+                                type: string
+                                description: |
+                                    The message body to publish.
+                                example: Your backups have failed.
+                            priority:
+                                type: string
+                                description: |
+                                    The priority to set.
+                                example: urgent
+                            tags:
+                                type: string
+                                description: |
+                                    Tags to attach to the message.
+                                example: incoming_envelope
+                    fail:
+                        type: object
+                        properties:
+                            title:
+                                type: string
+                                description: |
+                                    The title of the message.
+                                example: Ping!
+                            message:
+                                type: string
+                                description: |
+                                    The message body to publish.
+                                example: Your backups have failed.
+                            priority:
+                                type: string
+                                description: |
+                                    The priority to set.
+                                example: urgent
+                            tags:
+                                type: string
+                                description: |
+                                    Tags to attach to the message.
+                                example: incoming_envelope
+                    states:
+                        type: array
+                        items:
+                            type: string
+                            enum:
+                                - start
+                                - finish
+                                - fail
+                            uniqueItems: true
+                        description: |
+                            List of one or more monitoring states to ping for:
+                            "start", "finish", and/or "fail". Defaults to
+                            pinging for failure only.
+                        example:
+                            - start
+                            - finish
             healthchecks:
                 type: object
                 required: ['ping_url']

borgmatic/config/validate.py

@@ -4,7 +4,7 @@ import jsonschema
 import pkg_resources
 import ruamel.yaml
 
-from borgmatic.config import load, normalize, override
+from borgmatic.config import environment, load, normalize, override
 
 
 def schema_filename():
@@ -79,7 +79,7 @@ def apply_logical_validation(config_filename, parsed_configuration):
             )
 
 
-def parse_configuration(config_filename, schema_filename, overrides=None):
+def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True):
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
     rendition of JSON Schema format, a sequence of configuration file override strings in the form
@@ -98,8 +98,10 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
     except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
-    override.apply_overrides(config, overrides)
     normalize.normalize(config)
+    override.apply_overrides(config, overrides)
+    if resolve_env:
+        environment.resolve_env_variables(config)
 
     try:
         validator = jsonschema.Draft7Validator(schema)

borgmatic/execute.py

@@ -87,18 +87,19 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
                             # Add the process's output to output_buffers to ensure it'll get read.
                             output_buffers.append(other_process.stdout)
 
-                line = ready_buffer.readline().rstrip().decode()
-                if not line or not ready_process:
-                    continue
+                while True:
+                    line = ready_buffer.readline().rstrip().decode()
+                    if not line or not ready_process:
+                        break
 
-                # Keep the last few lines of output in case the process errors, and we need the output for
-                # the exception below.
-                last_lines = buffer_last_lines[ready_buffer]
-                last_lines.append(line)
-                if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
-                    last_lines.pop(0)
+                    # Keep the last few lines of output in case the process errors, and we need the output for
+                    # the exception below.
+                    last_lines = buffer_last_lines[ready_buffer]
+                    last_lines.append(line)
+                    if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+                        last_lines.pop(0)
 
-                logger.log(output_log_level, line)
+                    logger.log(output_log_level, line)
 
         still_running = False
 
@@ -132,21 +133,6 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
         if not still_running:
             break
 
-    # Consume any remaining output that we missed (if any).
-    for process in processes:
-        output_buffer = output_buffer_for_process(process, exclude_stdouts)
-
-        if not output_buffer:
-            continue
-
-        while True:  # pragma: no cover
-            remaining_output = output_buffer.readline().rstrip().decode()
-
-            if not remaining_output:
-                break
-
-            logger.log(output_log_level, remaining_output)
-
 
 def log_command(full_command, input_file, output_file):
     '''

borgmatic/hooks/cronhub.py

@@ -43,7 +43,9 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)
         try:
-            requests.get(ping_url)
+            response = requests.get(ping_url)
+            if not response.ok:
+                response.raise_for_status()
         except requests.exceptions.RequestException as error:
             logger.warning(f'{config_filename}: Cronhub error: {error}')
 

borgmatic/hooks/cronitor.py

@@ -38,7 +38,9 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)
         try:
-            requests.get(ping_url)
+            response = requests.get(ping_url)
+            if not response.ok:
+                response.raise_for_status()
         except requests.exceptions.RequestException as error:
             logger.warning(f'{config_filename}: Cronitor error: {error}')
 

borgmatic/hooks/dispatch.py

@@ -1,17 +1,27 @@
 import logging
 
-from borgmatic.hooks import cronhub, cronitor, healthchecks, mongodb, mysql, pagerduty, postgresql
+from borgmatic.hooks import (
+    cronhub,
+    cronitor,
+    healthchecks,
+    mongodb,
+    mysql,
+    ntfy,
+    pagerduty,
+    postgresql,
+)
 
 logger = logging.getLogger(__name__)
 
 HOOK_NAME_TO_MODULE = {
-    'healthchecks': healthchecks,
-    'cronitor': cronitor,
     'cronhub': cronhub,
+    'cronitor': cronitor,
+    'healthchecks': healthchecks,
+    'mongodb_databases': mongodb,
+    'mysql_databases': mysql,
+    'ntfy': ntfy,
     'pagerduty': pagerduty,
     'postgresql_databases': postgresql,
-    'mysql_databases': mysql,
-    'mongodb_databases': mongodb,
 }
 
 

borgmatic/hooks/healthchecks.py

@@ -125,7 +125,9 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)
         try:
-            requests.post(ping_url, data=payload.encode('utf-8'))
+            response = requests.post(ping_url, data=payload.encode('utf-8'))
+            if not response.ok:
+                response.raise_for_status()
         except requests.exceptions.RequestException as error:
             logger.warning(f'{config_filename}: Healthchecks error: {error}')
 

borgmatic/hooks/monitor.py

@@ -1,6 +1,6 @@
 from enum import Enum
 
-MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty')
+MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy')
 
 
 class State(Enum):

borgmatic/hooks/ntfy.py

@@ -0,0 +1,84 @@
+import logging
+
+import requests
+
+from borgmatic.hooks import monitor
+
+logger = logging.getLogger(__name__)
+
+MONITOR_STATE_TO_NTFY = {
+    monitor.State.START: None,
+    monitor.State.FINISH: None,
+    monitor.State.FAIL: None,
+}
+
+
+def initialize_monitor(
+    ping_url, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
+def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
+    '''
+    Ping the configured Ntfy topic. Use the given configuration filename in any log entries.
+    If this is a dry run, then don't actually ping anything.
+    '''
+
+    run_states = hook_config.get('states', ['fail'])
+
+    if state.name.lower() in run_states:
+        dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+
+        state_config = hook_config.get(
+            state.name.lower(),
+            {
+                'title': f'A Borgmatic {state.name} event happened',
+                'message': f'A Borgmatic {state.name} event happened',
+                'priority': 'default',
+                'tags': 'borgmatic',
+            },
+        )
+
+        base_url = hook_config.get('server', 'https://ntfy.sh')
+        topic = hook_config.get('topic')
+
+        logger.info(f'{config_filename}: Pinging ntfy topic {topic}{dry_run_label}')
+        logger.debug(f'{config_filename}: Using Ntfy ping URL {base_url}/{topic}')
+
+        headers = {
+            'X-Title': state_config.get('title'),
+            'X-Message': state_config.get('message'),
+            'X-Priority': state_config.get('priority'),
+            'X-Tags': state_config.get('tags'),
+        }
+
+        if (
+            base_url != 'https://ntfy.sh'
+            and hook_config.get('username') is not None
+            and hook_config.get('password') is not None
+        ):
+            headers.update(
+                {'Authorization': f'{hook_config.get("username")}:{hook_config.get("password")}'}
+            )
+
+        if not dry_run:
+            logging.getLogger('urllib3').setLevel(logging.ERROR)
+            try:
+                response = requests.post(f'{base_url}/{topic}', headers=headers)
+                if not response.ok:
+                    response.raise_for_status()
+            except requests.exceptions.RequestException as error:
+                logger.warning(f'{config_filename}: Ntfy error: {error}')
+
+
+def destroy_monitor(
+    ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No destruction is necessary for this monitor.
+    '''
+    pass

borgmatic/hooks/pagerduty.py

@@ -69,7 +69,9 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
 
     logging.getLogger('urllib3').setLevel(logging.ERROR)
     try:
-        requests.post(EVENTS_API_URL, data=payload.encode('utf-8'))
+        response = requests.post(EVENTS_API_URL, data=payload.encode('utf-8'))
+        if not response.ok:
+            response.raise_for_status()
     except requests.exceptions.RequestException as error:
         logger.warning(f'{config_filename}: PagerDuty error: {error}')
 

docs/Dockerfile

@@ -1,14 +1,14 @@
-FROM python:3.8-alpine3.13 as borgmatic
+FROM alpine:3.16.0 as borgmatic
 
 COPY . /app
-RUN apk add --no-cache py3-ruamel.yaml py3-ruamel.yaml.clib
+RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
     && for action in init prune compact create check extract export-tar mount umount restore list info borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:15.2.1-alpine as html
+FROM node:18.4.0-alpine as html
 
 ARG ENVIRONMENT=production
 
@@ -27,7 +27,7 @@ COPY . /source
 RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.19.4-alpine
+FROM nginx:1.22.0-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -3,7 +3,7 @@ title: How to add preparation and cleanup steps to backups
 eleventyNavigation:
   key: 🧹 Add preparation and cleanup steps
   parent: How-to guides
-  order: 8
+  order: 9
 ---
 ## Preparation and cleanup hooks
 
@@ -28,7 +28,8 @@ hooks:
         - umount /some/filesystem
 ```
 
-The `before_backup` and `after_backup` hooks each run once per repository in a
+<span class="minilink minilink-addedin">New in version 1.6.0</span> The
+`before_backup` and `after_backup` hooks each run once per repository in a
 configuration file. `before_backup` hooks runs right before the `create`
 action for a particular repository, and `after_backup` hooks run afterwards,
 but not if an error occurs in a previous hook or in the backups themselves.
@@ -61,6 +62,10 @@ variables you can use here:
  * `repository`: path of the current repository as configured in the current
    borgmatic configuration file
 
+Note that you can also interpolate in [arbitrary environment
+variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).
+
+
 ## Global hooks
 
 You can also use `before_everything` and `after_everything` hooks to perform

docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md

@@ -3,7 +3,7 @@ title: How to backup to a removable drive or an intermittent server
 eleventyNavigation:
   key: 💾 Backup to a removable drive/server
   parent: How-to guides
-  order: 9
+  order: 10
 ---
 ## Occasional backups
 

docs/how-to/backup-your-databases.md

@@ -3,7 +3,7 @@ title: How to backup your databases
 eleventyNavigation:
   key: 🗄️ Backup your databases
   parent: How-to guides
-  order: 7
+  order: 8
 ---
 ## Database dump hooks
 
@@ -100,6 +100,14 @@ hooks:
         - name: all
 ```
 
+### External passwords
+
+If you don't want to keep your database passwords in your borgmatic
+configuration file, you can instead pass them in via [environment
+variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/)
+or command-line [configuration
+overrides](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides).
+
 
 ### Configuration backups
 

docs/how-to/deal-with-very-large-backups.md

@@ -3,7 +3,7 @@ title: How to deal with very large backups
 eleventyNavigation:
   key: 📏 Deal with very large backups
   parent: How-to guides
-  order: 3
+  order: 4
 ---
 ## Biggish data
 
@@ -74,8 +74,9 @@ See [Borg's check documentation](https://borgbackup.readthedocs.io/en/stable/usa
 
 ### Check frequency
 
-As of borgmatic 1.6.2, you can optionally configure checks to run on a
-periodic basis rather than every time borgmatic runs checks. For instance:
+<span class="minilink minilink-addedin">New in version 1.6.2</span> You can
+optionally configure checks to run on a periodic basis rather than every time
+borgmatic runs checks. For instance:
 
 ```yaml
 consistency:
@@ -96,6 +97,10 @@ within `~/.borgmatic/checks`). If it hasn't been long enough, the check is
 skipped. And you still have to run `borgmatic check` (or just `borgmatic`) in
 order for checks to run, even when a `frequency` is configured!
 
+This also applies *across* configuration files that have the same repository
+configured. Just make sure you have the same check frequency configured in
+each—or the most frequently configured check will apply.
+
 If you want to temporarily ignore your configured frequencies, you can invoke
 `borgmatic check --force` to run checks unconditionally.
 

docs/how-to/develop-on-borgmatic.md

@@ -3,7 +3,7 @@ title: How to develop on borgmatic
 eleventyNavigation:
   key: 🏗️ Develop on borgmatic
   parent: How-to guides
-  order: 12
+  order: 13
 ---
 ## Source code
 

docs/how-to/extract-a-backup.md

@@ -3,7 +3,7 @@ title: How to extract a backup
 eleventyNavigation:
   key: 📤 Extract a backup
   parent: How-to guides
-  order: 6
+  order: 7
 ---
 ## Extract
 

docs/how-to/inspect-your-backups.md

@@ -3,7 +3,7 @@ title: How to inspect your backups
 eleventyNavigation:
   key: 🔎 Inspect your backups
   parent: How-to guides
-  order: 4
+  order: 5
 ---
 ## Backup progress
 
@@ -53,10 +53,10 @@ borgmatic info
 
 ### Searching for a file
 
-Let's say you've accidentally deleted a file and want to find the backup
-archive(s) containing it. `borgmatic list` provides a `--find` flag for
-exactly this purpose (as of borgmatic 1.6.3). For instance, if you're looking
-for a `foo.txt`:
+<span class="minilink minilink-addedin">New in version 1.6.3</span> Let's say
+you've accidentally deleted a file and want to find the backup archive(s)
+containing it. `borgmatic list` provides a `--find` flag for exactly this
+purpose. For instance, if you're looking for a `foo.txt`:
 
 ```bash
 borgmatic list --find foo.txt

docs/how-to/make-backups-redundant.md

@@ -3,7 +3,7 @@ title: How to make backups redundant
 eleventyNavigation:
   key: ☁️  Make backups redundant
   parent: How-to guides
-  order: 2
+  order: 3
 ---
 ## Multiple repositories
 

docs/how-to/make-per-application-backups.md

@@ -123,11 +123,7 @@ Once this include gets merged in, the resulting configuration would have a
 `keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
 
 When there's an option collision between the local file and the merged
-include, the local file's option takes precedence. And as of borgmatic 1.6.0,
-this feature performs a deep merge, meaning that values are merged at all
-levels in the two configuration files. Colliding list values are appended
-together. This allows you to include common configuration—up to full borgmatic
-configuration files—while overriding only the parts you want to customize.
+include, the local file's option takes precedence.
 
 Note that this `<<` include merging syntax is only for merging in mappings
 (configuration options and their values). But if you'd like to include a
@@ -139,6 +135,16 @@ global level, another `<<` within each configuration section, etc. (This is a
 YAML limitation.)
 
 
+### Deep merge
+
+<span class="minilink minilink-addedin">New in version 1.6.0</span> borgmatic
+performs a deep merge of merged include files, meaning that values are merged
+at all levels in the two configuration files. Colliding list values are
+appended together. This allows you to include common configuration—up to full
+borgmatic configuration files—while overriding only the parts you want to
+customize.
+
+
 ## Configuration overrides
 
 In more complex multi-application setups, you may want to override particular
@@ -203,3 +209,5 @@ indentation and a leading dash.)
 
 Be sure to quote your overrides if they contain spaces or other characters
 that your shell may interpret.
+
+An alternate to command-line overrides is passing in your values via [environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).

docs/how-to/monitor-your-backups.md

@@ -3,7 +3,7 @@ title: How to monitor your backups
 eleventyNavigation:
   key: 🚨 Monitor your backups
   parent: How-to guides
-  order: 5
+  order: 6
 ---
 
 ## Monitoring and alerting
@@ -38,17 +38,19 @@ below for how to configure this.
 
 borgmatic integrates with monitoring services like
 [Healthchecks](https://healthchecks.io/), [Cronitor](https://cronitor.io),
-[Cronhub](https://cronhub.io), and [PagerDuty](https://www.pagerduty.com/) and
-pings these services whenever borgmatic runs. That way, you'll receive an
-alert when something goes wrong or (for certain hooks) the service doesn't
-hear from borgmatic for a configured interval. See [Healthchecks
+[Cronhub](https://cronhub.io), [PagerDuty](https://www.pagerduty.com/), and
+[ntfy](https://ntfy.sh/) and pings these services whenever borgmatic runs.
+That way, you'll receive an alert when something goes wrong or (for certain
+hooks) the service doesn't hear from borgmatic for a configured interval. See
+[Healthchecks
 hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook),
 [Cronitor
 hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook),
 [Cronhub
 hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook),
-and [PagerDuty
-hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook)
+[PagerDuty
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook),
+and [ntfy hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook)
 below for how to configure this.
 
 While these services offer different features, you probably only need to use
@@ -59,8 +61,6 @@ one of them at most.
 You can use traditional monitoring software to consume borgmatic JSON output
 and track when the last successful backup occurred. See [scripting
 borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic)
-and [related
-software](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#related-software)
 below for how to configure this.
 
 ### Borg hosting providers
@@ -270,6 +270,58 @@ If you have any issues with the integration, [please contact
 us](https://torsion.org/borgmatic/#support-and-contributing).
 
 
+## ntfy hook
+
+[ntfy](https://ntfy.sh) is a free, simple, service (either hosted or self-hosted)
+which offers simple pub/sub push notifications to multiple platforms including
+[web](https://ntfy.sh/stats), [Android](https://play.google.com/store/apps/details?id=io.heckel.ntfy)
+and [iOS](https://apps.apple.com/us/app/ntfy/id1625396347).
+
+Since push notifications for regular events might soon become quite annoying,
+this hook only fires on any errors by default in order to instantly alert you to issues.
+The `states` list can override this.
+
+As ntfy is unauthenticated, it isn't a suitable channel for any private information
+so the default messages are intentionally generic. These can be overridden, depending
+on your risk assessment. Each `state` can have its own custom messages, priorities and tags
+or, if none are provided, will use the default.
+
+A [self-hosted ntfy instance](https://ntfy.sh/docs/install/) can be used if desired,
+including support for Basic Authentication using the `username` and `password` fields
+with a custom `server` value
+
+An example configuration is shown here, with all the available options, including
+[priorities](https://ntfy.sh/docs/publish/#message-priority) and
+[tags](https://ntfy.sh/docs/publish/#tags-emojis):
+
+```yaml
+hooks:
+    ntfy:
+        topic: my-unique-topic
+        server: https://ntfy.my-domain.com
+        username: borgmatic
+        password: superSecretP@ssw0rd
+        start:
+            title: A Borgmatic backup started
+            message: Watch this space...
+            tags: borgmatic
+            priority: min
+        finish:
+            title: A Borgmatic backup completed successfully
+            message: Nice!
+            tags: borgmatic,+1
+            priority: min
+        fail:
+            title: A Borgmatic backup failed
+            message: You should probably fix it
+            tags: borgmatic,-1,skull
+            priority: max
+        states:
+            - start
+            - finish
+            - fail
+```
+
 ## Scripting borgmatic
 
 To consume the output of borgmatic in other software, you can include an
@@ -281,11 +333,6 @@ suppressed so as not to interfere with the captured JSON. Also note that JSON
 output only shows up at the console, and not in syslog.
 
 
-## Related software
-
- * [Borgmacator GNOME AppIndicator](https://github.com/N-Coder/borgmacator/)
-
-
 ### Latest backups
 
 All borgmatic actions that accept an "--archive" flag allow you to specify an

docs/how-to/provide-your-passwords.md

@@ -0,0 +1,90 @@
+---
+title: How to provide your passwords
+eleventyNavigation:
+  key: 🔒 Provide your passwords
+  parent: How-to guides
+  order: 2
+---
+## Environment variable interpolation
+
+If you want to use a Borg repository passphrase or database passwords with
+borgmatic, you can set them directly in your borgmatic configuration file,
+treating those secrets like any other option value. But if you'd rather store
+them outside of borgmatic, whether for convenience or security reasons, read
+on.
+
+<span class="minilink minilink-addedin">New in version 1.6.4</span> borgmatic
+supports interpolating arbitrary environment variables directly into option
+values in your configuration file. That means you can instruct borgmatic to
+pull your repository passphrase, your database passwords, or any other option
+values from environment variables. For instance:
+
+```yaml
+storage:
+    encryption_passphrase: ${MY_PASSPHRASE}
+```
+
+This uses the `MY_PASSPHRASE` environment variable as your encryption
+passphrase. Note that the `{` `}` brackets are required. Just `$MY_PASSPHRASE`
+will not work.
+
+In the case of `encryption_passphrase` in particular, an alternate approach
+is to use Borg's `BORG_PASSPHRASE` environment variable, which doesn't even
+require setting an explicit `encryption_passphrase` value in borgmatic's
+configuration file.
+
+For [database
+configuration](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/),
+the same approach applies. For example:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: users
+          password: ${MY_DATABASE_PASSWORD}
+```
+
+This uses the `MY_DATABASE_PASSWORD` environment variable as your database
+password.
+
+
+### Interpolation defaults
+
+If you'd like to set a default for your environment variables, you can do so with the following syntax:
+
+```yaml
+storage:
+    encryption_passphrase: ${MY_PASSPHRASE:-defaultpass}
+```
+
+Here, "`defaultpass`" is the default passphrase if the `MY_PASSPHRASE`
+environment variable is not set. Without a default, if the environment
+variable doesn't exist, borgmatic will error.
+
+
+### Disabling interpolation
+
+To disable this environment variable interpolation feature entirely, you can
+pass the `--no-environment-interpolation` flag on the command-line.
+
+Or if you'd like to disable interpolation within a single option value, you
+can escape it with a backslash. For instance, if your password is literally
+`${A}@!`:
+
+```yaml
+storage:
+    encryption_passphrase: \${A}@!
+```
+
+### Related features
+
+Another way to override particular options within a borgmatic configuration
+file is to use a [configuration
+override](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides)
+on the command-line. But please be aware of the security implications of
+specifying secrets on the command-line.
+
+Additionally, borgmatic action hooks support their own [variable
+interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation),
+although in that case it's for particular borgmatic runtime values rather than
+(only) environment variables.

docs/how-to/run-arbitrary-borg-commands.md

@@ -3,7 +3,7 @@ title: How to run arbitrary Borg commands
 eleventyNavigation:
   key: 🔧 Run arbitrary Borg commands
   parent: How-to guides
-  order: 10
+  order: 11
 ---
 ## Running Borg with borgmatic
 

docs/how-to/set-up-backups.md

@@ -226,24 +226,37 @@ good idea to test that borgmatic is working. So to run borgmatic and start a
 backup, you can invoke it like this:
 
 ```bash
-sudo borgmatic --verbosity 1 --files
+sudo borgmatic create --verbosity 1 --files --stats
 ```
 
 (No borgmatic `--files` flag? It's only present in newer versions of
 borgmatic. So try leaving it out, or upgrade borgmatic!)
 
-By default, this will also prune any old backups as per the configured
-retention policy, compact segments to free up space (with Borg 1.2+), and
-check backups for consistency problems due to things like file damage.
+The `--verbosity` flag makes borgmatic show the steps it's performing. The
+`--files` flag lists each file that's new or changed since the last backup.
+And `--stats` shows summary information about the created archive. All of
+these flags are optional.
 
-The verbosity flag makes borgmatic show the steps it's performing. And the
-files flag lists each file that's new or changed since the last backup.
-Eyeball the list and see if it matches your expectations based on the
-configuration.
+As the command runs, you should eyeball the output to see if it matches your
+expectations based on your configuration.
 
 If you'd like to specify an alternate configuration file path, use the
-`--config` flag. See `borgmatic --help` for more information.
+`--config` flag.
 
+See `borgmatic --help` and `borgmatic create --help` for more information.
+
+
+## Default actions
+
+If you omit `create` and other actions, borgmatic runs through a set of
+default actions: `prune` any old backups as per the configured retention
+policy, `compact` segments to free up space (with Borg 1.2+), `create` a
+backup, *and* `check` backups for consistency problems due to things like file
+damage. For instance:
+
+```bash
+sudo borgmatic --verbosity 1 --files --stats
+```
 
 ## Autopilot
 

docs/how-to/upgrade.md

@@ -3,7 +3,7 @@ title: How to upgrade borgmatic
 eleventyNavigation:
   key: 📦 Upgrade borgmatic
   parent: How-to guides
-  order: 11
+  order: 12
 ---
 ## Upgrading
 

scripts/release

@@ -21,6 +21,12 @@ if [[ $version =~ .*dev* ]]; then
     exit 1
 fi
 
+if ! git diff-index --quiet HEAD -- ; then
+    echo "Refusing to release with local changes:"
+    git status --porcelain
+    exit 1
+fi
+
 git tag $version
 git push origin $version
 git push github $version

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.6.3.dev0'
+VERSION = '1.6.5'
 
 
 setup(

tests/integration/config/test_generate.py

@@ -113,7 +113,7 @@ def test_write_configuration_with_already_existing_file_raises():
 def test_write_configuration_with_already_existing_file_and_overwrite_does_not_raise():
     flexmock(os.path).should_receive('exists').and_return(True)
 
-    module.write_configuration('config.yaml', 'config: yaml', overwrite=True)
+    module.write_configuration('/tmp/config.yaml', 'config: yaml', overwrite=True)
 
 
 def test_write_configuration_with_already_existing_directory_does_not_raise():

tests/integration/test_execute.py

@@ -70,6 +70,27 @@ def test_log_outputs_includes_error_output_in_exception():
     assert error.value.output
 
 
+def test_log_outputs_logs_multiline_error_output():
+    '''
+    Make sure that all error output lines get logged, not just (for instance) the first few lines
+    of a process' traceback.
+    '''
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
+
+    process = subprocess.Popen(
+        ['python', '-c', 'foopydoo'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
+    flexmock(module.logger).should_call('log').at_least().times(3)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        module.log_outputs(
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
+        )
+
+
 def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdout():
     flexmock(module.logger).should_receive('log')
     flexmock(module).should_receive('exit_code_indicates_error').and_return(True)

tests/unit/borg/test_borg.py

@@ -8,8 +8,12 @@ from ..test_verbosity import insert_logging_mock
 
 
 def test_run_arbitrary_borg_calls_borg_with_parameters():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'break-lock', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
+        ('borg', 'break-lock', 'repo'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -18,10 +22,12 @@ def test_run_arbitrary_borg_calls_borg_with_parameters():
 
 
 def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo', '--info'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -31,10 +37,12 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter():
 
 
 def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo', '--debug', '--show-rc'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -45,10 +53,12 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters():
     storage_config = {'lock_wait': 5}
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo', '--lock-wait', '5'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -58,10 +68,12 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters(
 
 def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter():
     storage_config = {}
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo::archive'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -70,8 +82,12 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter():
 
 
 def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'break-lock', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
+        ('borg1', 'break-lock', 'repo'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg1',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -80,10 +96,12 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
 
 
 def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_parameters():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo', '--remote-path', 'borg1'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -92,10 +110,12 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet
 
 
 def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo', '--progress'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -104,8 +124,12 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg():
 
 
 def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'break-lock', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg',
+        ('borg', 'break-lock', 'repo'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -114,8 +138,9 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg():
 
 
 def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg',), output_log_level=logging.WARNING, borg_local_path='borg',
+        ('borg',), output_log_level=logging.WARNING, borg_local_path='borg', extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -124,8 +149,12 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise():
 
 
 def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'key', 'export', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg',
+        ('borg', 'key', 'export', 'repo'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -134,10 +163,12 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository():
 
 
 def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'debug', 'dump-manifest', 'repo', 'path'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -146,8 +177,12 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository()
 
 
 def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repository():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'debug', 'info'), output_log_level=logging.WARNING, borg_local_path='borg',
+        ('borg', 'debug', 'info'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(
@@ -156,10 +191,12 @@ def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repositor
 
 
 def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_borg_repository():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'debug', 'convert-profile', 'in', 'out'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.run_arbitrary_borg(

tests/unit/borg/test_check.py

@@ -9,7 +9,10 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(command):
-    flexmock(module).should_receive('execute_command').with_args(command).once()
+    flexmock(module.environment).should_receive('make_environment')
+    flexmock(module).should_receive('execute_command').with_args(
+        command, extra_environment=None
+    ).once()
 
 
 def insert_execute_command_never():
@@ -310,8 +313,11 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter():
     )
     flexmock(module).should_receive('make_check_flags').and_return(())
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'check', '--progress', 'repo'), output_file=module.DO_NOT_CAPTURE
+        ('borg', 'check', '--progress', 'repo'),
+        output_file=module.DO_NOT_CAPTURE,
+        extra_environment=None,
     ).once()
     flexmock(module).should_receive('make_check_time_path')
     flexmock(module).should_receive('write_check_time')
@@ -335,8 +341,11 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter():
     )
     flexmock(module).should_receive('make_check_flags').and_return(())
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'check', '--repair', 'repo'), output_file=module.DO_NOT_CAPTURE
+        ('borg', 'check', '--repair', 'repo'),
+        output_file=module.DO_NOT_CAPTURE,
+        extra_environment=None,
     ).once()
     flexmock(module).should_receive('make_check_time_path')
     flexmock(module).should_receive('write_check_time')

tests/unit/borg/test_compact.py

@@ -8,8 +8,12 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(compact_command, output_log_level):
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        compact_command, output_log_level=output_log_level, borg_local_path=compact_command[0]
+        compact_command,
+        output_log_level=output_log_level,
+        borg_local_path=compact_command[0],
+        extra_environment=None,
     ).once()
 
 

tests/unit/borg/test_create.py

@@ -292,12 +292,50 @@ def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        local_borg_version='1.2.3',
+    )
+
+
+def test_create_archive_calls_borg_with_environment():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
+    flexmock(module).should_receive('expand_directories').and_return(())
+    flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
+    flexmock(module).should_receive('expand_home_directories').and_return(())
+    flexmock(module).should_receive('write_pattern_file').and_return(None)
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module).should_receive('ensure_files_readable')
+    flexmock(module).should_receive('make_pattern_flags').and_return(())
+    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    environment = {'BORG_THINGY': 'YUP'}
+    flexmock(module.environment).should_receive('make_environment').and_return(environment)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+        working_directory=None,
+        extra_environment=environment,
     )
 
     module.create_archive(
@@ -328,12 +366,14 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(pattern_flags)
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + pattern_flags + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -364,12 +404,14 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(exclude_flags)
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + exclude_flags + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -397,12 +439,14 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -431,12 +475,14 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -466,12 +512,14 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--debug', '--show-rc') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -500,12 +548,14 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -535,12 +585,14 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--dry-run') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -570,12 +622,14 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info', '--dry-run') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -605,12 +659,14 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--checkpoint-interval', '600') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -638,12 +694,14 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--chunker-params', '1,2,3,4') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -671,12 +729,14 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--compression', 'rle') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -709,12 +769,14 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_upload_ratelimit_
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', option_flag, '100') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -744,12 +806,14 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory='/working/dir',
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -778,12 +842,14 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--one-file-system') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -817,12 +883,14 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_ids_parameter
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', option_flag) + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -851,12 +919,14 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -892,12 +962,14 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + ((option_flag,) if option_flag else ()) + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -937,12 +1009,14 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + ((option_flag,) if option_flag else ()) + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -982,12 +1056,14 @@ def test_create_archive_with_bsd_flags_option_calls_borg_with_corresponding_para
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + ((option_flag,) if option_flag else ()) + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1016,12 +1092,14 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--files-cache', 'ctime,size') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1050,12 +1128,14 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg1',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1084,12 +1164,14 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--remote-path', 'borg1') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1118,12 +1200,14 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--umask', '740') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1151,12 +1235,14 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--lock-wait', '5') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1184,12 +1270,14 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--stats') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.WARNING,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1218,12 +1306,14 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info', '--stats') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -1253,12 +1343,14 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--list', '--filter', 'AME-') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.WARNING,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1287,12 +1379,14 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--list', '--filter', 'AME-', '--info') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -1322,12 +1416,14 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info', '--progress') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=module.DO_NOT_CAPTURE,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -1357,12 +1453,14 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--progress') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=module.DO_NOT_CAPTURE,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1392,6 +1490,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         ('borg', 'create', '--one-file-system', '--read-special', '--progress')
         + ARCHIVE_WITH_PATHS,
@@ -1400,6 +1499,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
         output_file=module.DO_NOT_CAPTURE,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1429,12 +1529,14 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -1465,12 +1567,14 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -1502,12 +1606,14 @@ def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
@@ -1536,12 +1642,14 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'),
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
@@ -1570,12 +1678,14 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1603,12 +1713,14 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'),
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1636,12 +1748,14 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'),
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1669,12 +1783,14 @@ def test_create_archive_with_repository_accepts_borg_placeholders():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '{fqdn}::Documents_{hostname}-{now}', 'foo', 'bar'),
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1702,12 +1818,14 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--extra', '--options') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(
@@ -1736,6 +1854,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes():
     flexmock(module).should_receive('ensure_files_readable')
     flexmock(module).should_receive('make_pattern_flags').and_return(())
     flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         ('borg', 'create', '--one-file-system', '--read-special') + ARCHIVE_WITH_PATHS,
         processes=processes,
@@ -1743,6 +1862,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes():
         output_file=None,
         borg_local_path='borg',
         working_directory=None,
+        extra_environment=None,
     )
 
     module.create_archive(

tests/unit/borg/test_environment.py

@@ -1,84 +1,34 @@
-import os
-
 from borgmatic.borg import environment as module
 
 
-def test_initialize_with_passcommand_should_set_environment():
-    orig_environ = os.environ
+def test_make_environment_with_passcommand_should_set_environment():
+    environment = module.make_environment({'encryption_passcommand': 'command'})
 
-    try:
-        os.environ = {}
-        module.initialize({'encryption_passcommand': 'command'})
-        assert os.environ.get('BORG_PASSCOMMAND') == 'command'
-    finally:
-        os.environ = orig_environ
+    assert environment.get('BORG_PASSCOMMAND') == 'command'
 
 
-def test_initialize_with_passphrase_should_set_environment():
-    orig_environ = os.environ
+def test_make_environment_with_passphrase_should_set_environment():
+    environment = module.make_environment({'encryption_passphrase': 'pass'})
 
-    try:
-        os.environ = {}
-        module.initialize({'encryption_passphrase': 'pass'})
-        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
-    finally:
-        os.environ = orig_environ
+    assert environment.get('BORG_PASSPHRASE') == 'pass'
 
 
-def test_initialize_with_ssh_command_should_set_environment():
-    orig_environ = os.environ
+def test_make_environment_with_ssh_command_should_set_environment():
+    environment = module.make_environment({'ssh_command': 'ssh -C'})
 
-    try:
-        os.environ = {}
-        module.initialize({'ssh_command': 'ssh -C'})
-        assert os.environ.get('BORG_RSH') == 'ssh -C'
-    finally:
-        os.environ = orig_environ
+    assert environment.get('BORG_RSH') == 'ssh -C'
 
 
-def test_initialize_without_configuration_should_only_set_default_environment():
-    orig_environ = os.environ
+def test_make_environment_without_configuration_should_only_set_default_environment():
+    environment = module.make_environment({})
 
-    try:
-        os.environ = {}
-        module.initialize({})
-
-        assert {key: value for key, value in os.environ.items() if key.startswith('BORG_')} == {
-            'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no',
-            'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no',
-        }
-    finally:
-        os.environ = orig_environ
+    assert environment == {
+        'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no',
+        'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no',
+    }
 
 
-def test_initialize_with_relocated_repo_access_should_override_default():
-    orig_environ = os.environ
+def test_make_environment_with_relocated_repo_access_should_override_default():
+    environment = module.make_environment({'relocated_repo_access_is_ok': True})
 
-    try:
-        os.environ = {}
-        module.initialize({'relocated_repo_access_is_ok': True})
-        assert os.environ.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'
-    finally:
-        os.environ = orig_environ
-
-
-def test_initialize_prefers_configuration_option_over_borg_environment_variable():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {'BORG_SSH': 'mosh'}
-        module.initialize({'ssh_command': 'ssh -C'})
-        assert os.environ.get('BORG_RSH') == 'ssh -C'
-    finally:
-        os.environ = orig_environ
-
-
-def test_initialize_passes_through_existing_borg_environment_variable():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {'BORG_PASSPHRASE': 'pass'}
-        module.initialize({'ssh_command': 'ssh -C'})
-        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
-    finally:
-        os.environ = orig_environ
+    assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'

tests/unit/borg/test_export_tar.py

@@ -10,11 +10,13 @@ from ..test_verbosity import insert_logging_mock
 def insert_execute_command_mock(
     command, output_log_level=logging.INFO, borg_local_path='borg', capture=True
 ):
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         command,
         output_file=None if capture else module.DO_NOT_CAPTURE,
         output_log_level=output_log_level,
         borg_local_path=borg_local_path,
+        extra_environment=None,
     ).once()
 
 

tests/unit/borg/test_extract.py

@@ -9,14 +9,16 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(command, working_directory=None):
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        command, working_directory=working_directory
+        command, working_directory=working_directory, extra_environment=None,
     ).once()
 
 
 def insert_execute_command_output_mock(command, result):
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        command, output_log_level=None, borg_local_path=command[0]
+        command, output_log_level=None, borg_local_path=command[0], extra_environment=None,
     ).and_return(result).once()
 
 
@@ -27,14 +29,14 @@ def test_extract_last_archive_dry_run_calls_borg_with_last_archive():
     insert_execute_command_mock(('borg', 'extract', '--dry-run', 'repo::archive2'))
     flexmock(module.feature).should_receive('available').and_return(True)
 
-    module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
+    module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=None)
 
 
 def test_extract_last_archive_dry_run_without_any_archives_should_not_raise():
     insert_execute_command_output_mock(('borg', 'list', '--short', 'repo'), result='\n')
     flexmock(module.feature).should_receive('available').and_return(True)
 
-    module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
+    module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=None)
 
 
 def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_parameter():
@@ -45,7 +47,7 @@ def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_paramet
     insert_logging_mock(logging.INFO)
     flexmock(module.feature).should_receive('available').and_return(True)
 
-    module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
+    module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=None)
 
 
 def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_parameter():
@@ -58,7 +60,7 @@ def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_param
     insert_logging_mock(logging.DEBUG)
     flexmock(module.feature).should_receive('available').and_return(True)
 
-    module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
+    module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=None)
 
 
 def test_extract_last_archive_dry_run_calls_borg_via_local_path():
@@ -68,7 +70,9 @@ def test_extract_last_archive_dry_run_calls_borg_via_local_path():
     insert_execute_command_mock(('borg1', 'extract', '--dry-run', 'repo::archive2'))
     flexmock(module.feature).should_receive('available').and_return(True)
 
-    module.extract_last_archive_dry_run(repository='repo', lock_wait=None, local_path='borg1')
+    module.extract_last_archive_dry_run(
+        storage_config={}, repository='repo', lock_wait=None, local_path='borg1'
+    )
 
 
 def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters():
@@ -80,7 +84,9 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters():
     )
     flexmock(module.feature).should_receive('available').and_return(True)
 
-    module.extract_last_archive_dry_run(repository='repo', lock_wait=None, remote_path='borg1')
+    module.extract_last_archive_dry_run(
+        storage_config={}, repository='repo', lock_wait=None, remote_path='borg1'
+    )
 
 
 def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters():
@@ -92,7 +98,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters():
     )
     flexmock(module.feature).should_receive('available').and_return(True)
 
-    module.extract_last_archive_dry_run(repository='repo', lock_wait=5)
+    module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=5)
 
 
 def test_extract_archive_calls_borg_with_path_parameters():
@@ -267,10 +273,12 @@ def test_extract_archive_calls_borg_with_strip_components():
 
 def test_extract_archive_calls_borg_with_progress_parameter():
     flexmock(module.os.path).should_receive('abspath').and_return('repo')
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'extract', '--progress', 'repo::archive'),
         output_file=module.DO_NOT_CAPTURE,
         working_directory=None,
+        extra_environment=None,
     ).once()
     flexmock(module.feature).should_receive('available').and_return(True)
 
@@ -306,11 +314,13 @@ def test_extract_archive_with_progress_and_extract_to_stdout_raises():
 def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
     flexmock(module.os.path).should_receive('abspath').and_return('repo')
     process = flexmock()
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'extract', '--stdout', 'repo::archive'),
         output_file=module.subprocess.PIPE,
         working_directory=None,
         run_to_completion=False,
+        extra_environment=None,
     ).and_return(process).once()
     flexmock(module.feature).should_receive('available').and_return(True)
 
@@ -331,8 +341,9 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
 
 def test_extract_archive_skips_abspath_for_remote_repository():
     flexmock(module.os.path).should_receive('abspath').never()
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'extract', 'server:repo::archive'), working_directory=None
+        ('borg', 'extract', 'server:repo::archive'), working_directory=None, extra_environment=None,
     ).once()
     flexmock(module.feature).should_receive('available').and_return(True)
 

tests/unit/borg/test_info.py

@@ -9,8 +9,12 @@ from ..test_verbosity import insert_logging_mock
 
 
 def test_display_archives_info_calls_borg_with_parameters():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
+        ('borg', 'info', 'repo'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.display_archives_info(
@@ -19,8 +23,12 @@ def test_display_archives_info_calls_borg_with_parameters():
 
 
 def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
+        ('borg', 'info', '--info', 'repo'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+        extra_environment=None,
     )
     insert_logging_mock(logging.INFO)
     module.display_archives_info(
@@ -29,8 +37,12 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
 
 
 def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_output():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
+        ('borg', 'info', '--json', 'repo'),
+        output_log_level=None,
+        borg_local_path='borg',
+        extra_environment=None,
     ).and_return('[]')
 
     insert_logging_mock(logging.INFO)
@@ -42,10 +54,12 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu
 
 
 def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--debug', '--show-rc', 'repo'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -55,8 +69,12 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
 
 
 def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_output():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
+        ('borg', 'info', '--json', 'repo'),
+        output_log_level=None,
+        borg_local_path='borg',
+        extra_environment=None,
     ).and_return('[]')
 
     insert_logging_mock(logging.DEBUG)
@@ -68,8 +86,12 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp
 
 
 def test_display_archives_info_with_json_calls_borg_with_json_parameter():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
+        ('borg', 'info', '--json', 'repo'),
+        output_log_level=None,
+        borg_local_path='borg',
+        extra_environment=None,
     ).and_return('[]')
 
     json_output = module.display_archives_info(
@@ -80,8 +102,12 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter():
 
 
 def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
+        ('borg', 'info', 'repo::archive'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.display_archives_info(
@@ -90,8 +116,12 @@ def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
 
 
 def test_display_archives_info_with_local_path_calls_borg_via_local_path():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
+        ('borg1', 'info', 'repo'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg1',
+        extra_environment=None,
     )
 
     module.display_archives_info(
@@ -103,10 +133,12 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path():
 
 
 def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_parameters():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--remote-path', 'borg1', 'repo'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.display_archives_info(
@@ -119,10 +151,12 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para
 
 def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters():
     storage_config = {'lock_wait': 5}
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--lock-wait', '5', 'repo'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.display_archives_info(
@@ -134,10 +168,12 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete
 
 @pytest.mark.parametrize('argument_name', ('prefix', 'glob_archives', 'sort_by', 'first', 'last'))
 def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     )
 
     module.display_archives_info(

tests/unit/borg/test_init.py

@@ -23,8 +23,12 @@ def insert_info_command_not_found_mock():
 
 
 def insert_init_command_mock(init_command, **kwargs):
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        init_command, output_file=module.DO_NOT_CAPTURE, borg_local_path=init_command[0]
+        init_command,
+        output_file=module.DO_NOT_CAPTURE,
+        borg_local_path=init_command[0],
+        extra_environment=None,
     ).once()
 
 
@@ -37,6 +41,7 @@ def test_initialize_repository_calls_borg_with_parameters():
 
 def test_initialize_repository_raises_for_borg_init_error():
     insert_info_command_not_found_mock()
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').and_raise(
         module.subprocess.CalledProcessError(2, 'borg init')
     )

tests/unit/borg/test_list.py

@@ -24,8 +24,12 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name():
 
 def test_resolve_archive_name_calls_borg_with_parameters():
     expected_archive = 'archive-name'
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        borg_local_path='borg',
+        extra_environment=None,
     ).and_return(expected_archive + '\n')
 
     assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
@@ -33,10 +37,12 @@ def test_resolve_archive_name_calls_borg_with_parameters():
 
 def test_resolve_archive_name_with_log_info_calls_borg_with_info_parameter():
     expected_archive = 'archive-name'
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--info') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
         borg_local_path='borg',
+        extra_environment=None,
     ).and_return(expected_archive + '\n')
     insert_logging_mock(logging.INFO)
 
@@ -45,10 +51,12 @@ def test_resolve_archive_name_with_log_info_calls_borg_with_info_parameter():
 
 def test_resolve_archive_name_with_log_debug_calls_borg_with_debug_parameter():
     expected_archive = 'archive-name'
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--debug', '--show-rc') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
         borg_local_path='borg',
+        extra_environment=None,
     ).and_return(expected_archive + '\n')
     insert_logging_mock(logging.DEBUG)
 
@@ -57,10 +65,12 @@ def test_resolve_archive_name_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
     expected_archive = 'archive-name'
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
         borg_local_path='borg1',
+        extra_environment=None,
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -71,10 +81,12 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
 
 def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_parameters():
     expected_archive = 'archive-name'
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
         borg_local_path='borg',
+        extra_environment=None,
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -84,8 +96,12 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param
 
 
 def test_resolve_archive_name_without_archives_raises():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        borg_local_path='borg',
+        extra_environment=None,
     ).and_return('')
 
     with pytest.raises(ValueError):
@@ -95,10 +111,12 @@ def test_resolve_archive_name_without_archives_raises():
 def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameters():
     expected_archive = 'archive-name'
 
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
         borg_local_path='borg',
+        extra_environment=None,
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -296,8 +314,12 @@ def test_list_archives_calls_borg_with_parameters():
         remote_path=None,
     ).and_return(('borg', 'list', 'repo'))
     flexmock(module).should_receive('make_find_paths').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
+        ('borg', 'list', 'repo'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+        extra_environment=None,
     ).once()
 
     module.list_archives(
@@ -316,8 +338,12 @@ def test_list_archives_with_json_suppresses_most_borg_output():
         remote_path=None,
     ).and_return(('borg', 'list', 'repo'))
     flexmock(module).should_receive('make_find_paths').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo'), output_log_level=None, borg_local_path='borg'
+        ('borg', 'list', 'repo'),
+        output_log_level=None,
+        borg_local_path='borg',
+        extra_environment=None,
     ).once()
 
     module.list_archives(
@@ -336,8 +362,12 @@ def test_list_archives_calls_borg_with_local_path():
         remote_path=None,
     ).and_return(('borg2', 'list', 'repo'))
     flexmock(module).should_receive('make_find_paths').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg2', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg2'
+        ('borg2', 'list', 'repo'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg2',
+        extra_environment=None,
     ).once()
 
     module.list_archives(
@@ -355,20 +385,27 @@ def test_list_archives_calls_borg_multiple_times_with_find_paths():
         ('borg', 'list', 'repo')
     ).and_return(('borg', 'list', 'repo::archive1')).and_return(('borg', 'list', 'repo::archive2'))
     flexmock(module).should_receive('make_find_paths').and_return(glob_paths)
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo'), output_log_level=None, borg_local_path='borg'
+        ('borg', 'list', 'repo'),
+        output_log_level=None,
+        borg_local_path='borg',
+        extra_environment=None,
     ).and_return(
         'archive1   Sun, 2022-05-29 15:27:04 [abc]\narchive2   Mon, 2022-05-30 19:47:15 [xyz]'
     ).once()
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive1') + glob_paths,
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive2') + glob_paths,
         output_log_level=logging.WARNING,
         borg_local_path='borg',
+        extra_environment=None,
     ).once()
 
     module.list_archives(
@@ -387,8 +424,12 @@ def test_list_archives_calls_borg_with_archive():
         remote_path=None,
     ).and_return(('borg', 'list', 'repo::archive'))
     flexmock(module).should_receive('make_find_paths').and_return(())
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
+        ('borg', 'list', 'repo::archive'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+        extra_environment=None,
     ).once()
 
     module.list_archives(

tests/unit/borg/test_mount.py

@@ -8,8 +8,9 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(command):
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        command, borg_local_path='borg'
+        command, borg_local_path='borg', extra_environment=None,
     ).once()
 
 
@@ -117,10 +118,12 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_parameters():
 
 
 def test_mount_archive_calls_borg_with_foreground_parameter():
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'),
         output_file=module.DO_NOT_CAPTURE,
         borg_local_path='borg',
+        extra_environment=None,
     ).once()
 
     module.mount_archive(

tests/unit/borg/test_prune.py

@@ -9,8 +9,12 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(prune_command, output_log_level):
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        prune_command, output_log_level=output_log_level, borg_local_path=prune_command[0]
+        prune_command,
+        output_log_level=output_log_level,
+        borg_local_path=prune_command[0],
+        extra_environment=None,
     ).once()
 
 

tests/unit/borg/test_version.py

@@ -11,39 +11,45 @@ VERSION = '1.2.3'
 
 
 def insert_execute_command_mock(command, borg_local_path='borg', version_output=f'borg {VERSION}'):
+    flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        command, output_log_level=None, borg_local_path=borg_local_path
+        command, output_log_level=None, borg_local_path=borg_local_path, extra_environment=None,
     ).once().and_return(version_output)
 
 
 def test_local_borg_version_calls_borg_with_required_parameters():
     insert_execute_command_mock(('borg', '--version'))
+    flexmock(module.environment).should_receive('make_environment')
 
-    assert module.local_borg_version() == VERSION
+    assert module.local_borg_version({}) == VERSION
 
 
 def test_local_borg_version_with_log_info_calls_borg_with_info_parameter():
     insert_execute_command_mock(('borg', '--version', '--info'))
     insert_logging_mock(logging.INFO)
+    flexmock(module.environment).should_receive('make_environment')
 
-    assert module.local_borg_version() == VERSION
+    assert module.local_borg_version({}) == VERSION
 
 
 def test_local_borg_version_with_log_debug_calls_borg_with_debug_parameters():
     insert_execute_command_mock(('borg', '--version', '--debug', '--show-rc'))
     insert_logging_mock(logging.DEBUG)
+    flexmock(module.environment).should_receive('make_environment')
 
-    assert module.local_borg_version() == VERSION
+    assert module.local_borg_version({}) == VERSION
 
 
 def test_local_borg_version_with_local_borg_path_calls_borg_with_it():
     insert_execute_command_mock(('borg1', '--version'), borg_local_path='borg1')
+    flexmock(module.environment).should_receive('make_environment')
 
-    assert module.local_borg_version('borg1') == VERSION
+    assert module.local_borg_version({}, 'borg1') == VERSION
 
 
 def test_local_borg_version_with_invalid_version_raises():
     insert_execute_command_mock(('borg', '--version'), version_output='wtf')
+    flexmock(module.environment).should_receive('make_environment')
 
     with pytest.raises(ValueError):
-        module.local_borg_version()
+        module.local_borg_version({})

tests/unit/commands/test_borgmatic.py

@@ -9,7 +9,6 @@ from borgmatic.commands import borgmatic as module
 
 
 def test_run_configuration_runs_actions_for_each_repository():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     expected_results = [flexmock(), flexmock()]
     flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return(
@@ -24,7 +23,6 @@ def test_run_configuration_runs_actions_for_each_repository():
 
 
 def test_run_configuration_with_invalid_borg_version_errors():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError)
     flexmock(module.command).should_receive('execute_hook').never()
     flexmock(module.dispatch).should_receive('call_hooks').never()
@@ -36,7 +34,6 @@ def test_run_configuration_with_invalid_borg_version_errors():
 
 
 def test_run_configuration_logs_monitor_start_error():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return(
         None
@@ -53,7 +50,6 @@ def test_run_configuration_logs_monitor_start_error():
 
 
 def test_run_configuration_bails_for_monitor_start_soft_failure():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
     flexmock(module.dispatch).should_receive('call_hooks').and_raise(error)
@@ -68,7 +64,6 @@ def test_run_configuration_bails_for_monitor_start_soft_failure():
 
 
 def test_run_configuration_logs_actions_error():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module.dispatch).should_receive('call_hooks')
@@ -84,7 +79,6 @@ def test_run_configuration_logs_actions_error():
 
 
 def test_run_configuration_bails_for_actions_soft_failure():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.dispatch).should_receive('call_hooks')
     error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
@@ -100,7 +94,6 @@ def test_run_configuration_bails_for_actions_soft_failure():
 
 
 def test_run_configuration_logs_monitor_finish_error():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
         None
@@ -117,7 +110,6 @@ def test_run_configuration_logs_monitor_finish_error():
 
 
 def test_run_configuration_bails_for_monitor_finish_soft_failure():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
     flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
@@ -135,7 +127,6 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure():
 
 
 def test_run_configuration_logs_on_error_hook_error():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook').and_raise(OSError)
     expected_results = [flexmock(), flexmock()]
@@ -152,7 +143,6 @@ def test_run_configuration_logs_on_error_hook_error():
 
 
 def test_run_configuration_bails_for_on_error_hook_soft_failure():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
     flexmock(module.command).should_receive('execute_hook').and_raise(error)
@@ -169,7 +159,6 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
 
 def test_run_configuration_retries_soft_error():
     # Run action first fails, second passes
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([])
@@ -182,7 +171,6 @@ def test_run_configuration_retries_soft_error():
 
 def test_run_configuration_retries_hard_error():
     # Run action fails twice
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
@@ -203,7 +191,6 @@ def test_run_configuration_retries_hard_error():
 
 
 def test_run_repos_ordered():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
@@ -221,7 +208,6 @@ def test_run_repos_ordered():
 
 
 def test_run_configuration_retries_round_robbin():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
@@ -252,7 +238,6 @@ def test_run_configuration_retries_round_robbin():
 
 
 def test_run_configuration_retries_one_passes():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
@@ -281,7 +266,6 @@ def test_run_configuration_retries_one_passes():
 
 
 def test_run_configuration_retry_wait():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
@@ -320,7 +304,6 @@ def test_run_configuration_retry_wait():
 
 
 def test_run_configuration_retries_timeout_multiple_repos():
-    flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(

tests/unit/config/test_environment.py

@@ -0,0 +1,87 @@
+import pytest
+
+from borgmatic.config import environment as module
+
+
+def test_env(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    config = {'key': 'Hello $MY_CUSTOM_VALUE'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello $MY_CUSTOM_VALUE'}
+
+
+def test_env_braces(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello foo'}
+
+
+def test_env_multi(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    monkeypatch.setenv('MY_CUSTOM_VALUE2', 'bar')
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE}${MY_CUSTOM_VALUE2}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello foobar'}
+
+
+def test_env_escape(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    monkeypatch.setenv('MY_CUSTOM_VALUE2', 'bar')
+    config = {'key': r'Hello ${MY_CUSTOM_VALUE} \${MY_CUSTOM_VALUE}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': r'Hello foo ${MY_CUSTOM_VALUE}'}
+
+
+def test_env_default_value(monkeypatch):
+    monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False)
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello bar'}
+
+
+def test_env_unknown(monkeypatch):
+    monkeypatch.delenv('MY_CUSTOM_VALUE', raising=False)
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
+    with pytest.raises(ValueError):
+        module.resolve_env_variables(config)
+
+
+def test_env_full(monkeypatch):
+    monkeypatch.setenv('MY_CUSTOM_VALUE', 'foo')
+    monkeypatch.delenv('MY_CUSTOM_VALUE2', raising=False)
+    config = {
+        'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
+        'dict': {
+            'key': 'value',
+            'anotherdict': {
+                'key': 'My ${MY_CUSTOM_VALUE} here',
+                'other': '${MY_CUSTOM_VALUE}',
+                'escaped': r'\${MY_CUSTOM_VALUE}',
+                'list': [
+                    '/home/${MY_CUSTOM_VALUE}/.local',
+                    '/var/log/',
+                    '/home/${MY_CUSTOM_VALUE2:-bar}/.config',
+                ],
+            },
+        },
+        'list': [
+            '/home/${MY_CUSTOM_VALUE}/.local',
+            '/var/log/',
+            '/home/${MY_CUSTOM_VALUE2-bar}/.config',
+        ],
+    }
+    module.resolve_env_variables(config)
+    assert config == {
+        'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
+        'dict': {
+            'key': 'value',
+            'anotherdict': {
+                'key': 'My foo here',
+                'other': 'foo',
+                'escaped': '${MY_CUSTOM_VALUE}',
+                'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'],
+            },
+        },
+        'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config'],
+    }

tests/unit/hooks/test_cronhub.py

@@ -5,7 +5,9 @@ from borgmatic.hooks import cronhub as module
 
 def test_ping_monitor_rewrites_ping_url_for_start_state():
     hook_config = {'ping_url': 'https://example.com/start/abcdef'}
-    flexmock(module.requests).should_receive('get').with_args('https://example.com/start/abcdef')
+    flexmock(module.requests).should_receive('get').with_args(
+        'https://example.com/start/abcdef'
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config,
@@ -18,7 +20,9 @@ def test_ping_monitor_rewrites_ping_url_for_start_state():
 
 def test_ping_monitor_rewrites_ping_url_and_state_for_start_state():
     hook_config = {'ping_url': 'https://example.com/ping/abcdef'}
-    flexmock(module.requests).should_receive('get').with_args('https://example.com/start/abcdef')
+    flexmock(module.requests).should_receive('get').with_args(
+        'https://example.com/start/abcdef'
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config,
@@ -31,7 +35,9 @@ def test_ping_monitor_rewrites_ping_url_and_state_for_start_state():
 
 def test_ping_monitor_rewrites_ping_url_for_finish_state():
     hook_config = {'ping_url': 'https://example.com/start/abcdef'}
-    flexmock(module.requests).should_receive('get').with_args('https://example.com/finish/abcdef')
+    flexmock(module.requests).should_receive('get').with_args(
+        'https://example.com/finish/abcdef'
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config,
@@ -44,7 +50,9 @@ def test_ping_monitor_rewrites_ping_url_for_finish_state():
 
 def test_ping_monitor_rewrites_ping_url_for_fail_state():
     hook_config = {'ping_url': 'https://example.com/start/abcdef'}
-    flexmock(module.requests).should_receive('get').with_args('https://example.com/fail/abcdef')
+    flexmock(module.requests).should_receive('get').with_args(
+        'https://example.com/fail/abcdef'
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
@@ -60,11 +68,32 @@ def test_ping_monitor_dry_run_does_not_hit_ping_url():
     )
 
 
-def test_ping_monitor_with_connection_error_does_not_raise():
+def test_ping_monitor_with_connection_error_logs_warning():
     hook_config = {'ping_url': 'https://example.com/start/abcdef'}
     flexmock(module.requests).should_receive('get').and_raise(
         module.requests.exceptions.ConnectionError
     )
+    flexmock(module.logger).should_receive('warning').once()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_other_error_logs_warning():
+    hook_config = {'ping_url': 'https://example.com/start/abcdef'}
+    response = flexmock(ok=False)
+    response.should_receive('raise_for_status').and_raise(
+        module.requests.exceptions.RequestException
+    )
+    flexmock(module.requests).should_receive('get').with_args(
+        'https://example.com/start/abcdef'
+    ).and_return(response)
+    flexmock(module.logger).should_receive('warning').once()
 
     module.ping_monitor(
         hook_config,

tests/unit/hooks/test_cronitor.py

@@ -5,7 +5,9 @@ from borgmatic.hooks import cronitor as module
 
 def test_ping_monitor_hits_ping_url_for_start_state():
     hook_config = {'ping_url': 'https://example.com'}
-    flexmock(module.requests).should_receive('get').with_args('https://example.com/run')
+    flexmock(module.requests).should_receive('get').with_args('https://example.com/run').and_return(
+        flexmock(ok=True)
+    )
 
     module.ping_monitor(
         hook_config,
@@ -18,7 +20,9 @@ def test_ping_monitor_hits_ping_url_for_start_state():
 
 def test_ping_monitor_hits_ping_url_for_finish_state():
     hook_config = {'ping_url': 'https://example.com'}
-    flexmock(module.requests).should_receive('get').with_args('https://example.com/complete')
+    flexmock(module.requests).should_receive('get').with_args(
+        'https://example.com/complete'
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config,
@@ -31,7 +35,9 @@ def test_ping_monitor_hits_ping_url_for_finish_state():
 
 def test_ping_monitor_hits_ping_url_for_fail_state():
     hook_config = {'ping_url': 'https://example.com'}
-    flexmock(module.requests).should_receive('get').with_args('https://example.com/fail')
+    flexmock(module.requests).should_receive('get').with_args(
+        'https://example.com/fail'
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
@@ -47,11 +53,32 @@ def test_ping_monitor_dry_run_does_not_hit_ping_url():
     )
 
 
-def test_ping_monitor_with_connection_error_does_not_raise():
+def test_ping_monitor_with_connection_error_logs_warning():
     hook_config = {'ping_url': 'https://example.com'}
     flexmock(module.requests).should_receive('get').and_raise(
         module.requests.exceptions.ConnectionError
     )
+    flexmock(module.logger).should_receive('warning').once()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_other_error_logs_warning():
+    hook_config = {'ping_url': 'https://example.com'}
+    response = flexmock(ok=False)
+    response.should_receive('raise_for_status').and_raise(
+        module.requests.exceptions.RequestException
+    )
+    flexmock(module.requests).should_receive('get').with_args('https://example.com/run').and_return(
+        response
+    )
+    flexmock(module.logger).should_receive('warning').once()
 
     module.ping_monitor(
         hook_config,

tests/unit/hooks/test_healthchecks.py

@@ -139,7 +139,7 @@ def test_ping_monitor_hits_ping_url_for_start_state():
     hook_config = {'ping_url': 'https://example.com'}
     flexmock(module.requests).should_receive('post').with_args(
         'https://example.com/start', data=''.encode('utf-8')
-    )
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config,
@@ -156,7 +156,7 @@ def test_ping_monitor_hits_ping_url_for_finish_state():
     flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
     flexmock(module.requests).should_receive('post').with_args(
         'https://example.com', data=payload.encode('utf-8')
-    )
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config,
@@ -173,7 +173,7 @@ def test_ping_monitor_hits_ping_url_for_fail_state():
     flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
     flexmock(module.requests).should_receive('post').with_args(
         'https://example.com/fail', data=payload.encode('utf')
-    )
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config,
@@ -190,7 +190,7 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url():
     flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
     flexmock(module.requests).should_receive('post').with_args(
         'https://hc-ping.com/{}'.format(hook_config['ping_url']), data=payload.encode('utf-8')
-    )
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config,
@@ -234,7 +234,7 @@ def test_ping_monitor_hits_ping_url_when_states_matching():
     hook_config = {'ping_url': 'https://example.com', 'states': ['start', 'finish']}
     flexmock(module.requests).should_receive('post').with_args(
         'https://example.com/start', data=''.encode('utf-8')
-    )
+    ).and_return(flexmock(ok=True))
 
     module.ping_monitor(
         hook_config,
@@ -245,13 +245,34 @@ def test_ping_monitor_hits_ping_url_when_states_matching():
     )
 
 
-def test_ping_monitor_with_connection_error_does_not_raise():
+def test_ping_monitor_with_connection_error_logs_warning():
     flexmock(module).should_receive('Forgetful_buffering_handler')
-    flexmock(module.logger).should_receive('warning')
     hook_config = {'ping_url': 'https://example.com'}
     flexmock(module.requests).should_receive('post').with_args(
         'https://example.com/start', data=''.encode('utf-8')
     ).and_raise(module.requests.exceptions.ConnectionError)
+    flexmock(module.logger).should_receive('warning').once()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        state=module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_other_error_logs_warning():
+    flexmock(module).should_receive('Forgetful_buffering_handler')
+    hook_config = {'ping_url': 'https://example.com'}
+    response = flexmock(ok=False)
+    response.should_receive('raise_for_status').and_raise(
+        module.requests.exceptions.RequestException
+    )
+    flexmock(module.requests).should_receive('post').with_args(
+        'https://example.com/start', data=''.encode('utf-8')
+    ).and_return(response)
+    flexmock(module.logger).should_receive('warning').once()
 
     module.ping_monitor(
         hook_config,

tests/unit/hooks/test_ntfy.py

@@ -0,0 +1,177 @@
+from enum import Enum
+
+from flexmock import flexmock
+
+from borgmatic.hooks import ntfy as module
+
+default_base_url = 'https://ntfy.sh'
+custom_base_url = 'https://ntfy.example.com'
+topic = 'borgmatic-unit-testing'
+
+custom_message_config = {
+    'title': 'Borgmatic unit testing',
+    'message': 'Borgmatic unit testing',
+    'priority': 'min',
+    'tags': '+1',
+}
+
+custom_message_headers = {
+    'X-Title': custom_message_config['title'],
+    'X-Message': custom_message_config['message'],
+    'X-Priority': custom_message_config['priority'],
+    'X-Tags': custom_message_config['tags'],
+}
+
+
+def return_default_message_headers(state=Enum):
+    headers = {
+        'X-Title': f'A Borgmatic {state.name} event happened',
+        'X-Message': f'A Borgmatic {state.name} event happened',
+        'X-Priority': 'default',
+        'X-Tags': 'borgmatic',
+    }
+    return headers
+
+
+def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{default_base_url}/{topic}',
+        headers=return_default_message_headers(module.monitor.State.FAIL),
+    ).and_return(flexmock(ok=True)).once()
+
+    module.ping_monitor(
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+    )
+
+
+def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').never()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').never()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.FINISH,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail():
+    hook_config = {'topic': topic, 'server': custom_base_url}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{custom_base_url}/{topic}',
+        headers=return_default_message_headers(module.monitor.State.FAIL),
+    ).and_return(flexmock(ok=True)).once()
+
+    module.ping_monitor(
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+    )
+
+
+def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').never()
+
+    module.ping_monitor(
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=True
+    )
+
+
+def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail():
+    hook_config = {'topic': topic, 'fail': custom_message_config}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{default_base_url}/{topic}', headers=custom_message_headers,
+    ).and_return(flexmock(ok=True)).once()
+
+    module.ping_monitor(
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+    )
+
+
+def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start():
+    hook_config = {'topic': topic, 'states': ['start', 'fail']}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{default_base_url}/{topic}',
+        headers=return_default_message_headers(module.monitor.State.START),
+    ).and_return(flexmock(ok=True)).once()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_connection_error_logs_warning():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{default_base_url}/{topic}',
+        headers=return_default_message_headers(module.monitor.State.FAIL),
+    ).and_raise(module.requests.exceptions.ConnectionError)
+    flexmock(module.logger).should_receive('warning').once()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.FAIL,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_other_error_logs_warning():
+    hook_config = {'topic': topic}
+    response = flexmock(ok=False)
+    response.should_receive('raise_for_status').and_raise(
+        module.requests.exceptions.RequestException
+    )
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{default_base_url}/{topic}',
+        headers=return_default_message_headers(module.monitor.State.FAIL),
+    ).and_return(response)
+    flexmock(module.logger).should_receive('warning').once()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.FAIL,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_minimal_config_hits_selfhosted_with_custom_auth():
+    hook_config = {
+        'topic': topic,
+        'server': custom_base_url,
+        'username': 'unittesting',
+        'password': 'dummypassword',
+    }
+    expected_headers = {
+        **return_default_message_headers(module.monitor.State.FAIL),
+        'Authorization': 'unittesting:dummypassword',
+    }
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{custom_base_url}/{topic}', headers=expected_headers,
+    ).and_return(flexmock(ok=True)).once()
+
+    module.ping_monitor(
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+    )

tests/unit/hooks/test_pagerduty.py

@@ -28,7 +28,7 @@ def test_ping_monitor_ignores_finish_state():
 
 
 def test_ping_monitor_calls_api_for_fail_state():
-    flexmock(module.requests).should_receive('post')
+    flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True))
 
     module.ping_monitor(
         {'integration_key': 'abc123'},
@@ -51,10 +51,27 @@ def test_ping_monitor_dry_run_does_not_call_api():
     )
 
 
-def test_ping_monitor_with_connection_error_does_not_raise():
+def test_ping_monitor_with_connection_error_logs_warning():
     flexmock(module.requests).should_receive('post').and_raise(
         module.requests.exceptions.ConnectionError
     )
+    flexmock(module.logger).should_receive('warning').once()
+
+    module.ping_monitor(
+        {'integration_key': 'abc123'},
+        'config.yaml',
+        module.monitor.State.FAIL,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_other_error_logs_warning():
+    response = flexmock(ok=False)
+    response.should_receive('raise_for_status').and_raise(
+        module.requests.exceptions.RequestException
+    )
+    flexmock(module.requests).should_receive('post').and_return(response)
     flexmock(module.logger).should_receive('warning')
 
     module.ping_monitor(