Reorganize data source and monitoring hooks to make developing new hooks easier.

NEWS

@@ -1,5 +1,6 @@
 1.9.4.dev0
  * #926: Fix library error when running within a PyInstaller bundle.
+ * Reorganize data source and monitoring hooks to make developing new hooks easier.
 
 1.9.3
  * #261 (beta): Add a ZFS hook for snapshotting and backing up ZFS datasets. See the documentation

borgmatic/actions/check.py

@@ -364,7 +364,7 @@ def collect_spot_check_source_paths(
             'use_streaming',
             config,
             repository['path'],
-            borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
         ).values()
     )
 

borgmatic/actions/create.py

@@ -10,7 +10,6 @@ import borgmatic.config.paths
 import borgmatic.config.validate
 import borgmatic.hooks.command
 import borgmatic.hooks.dispatch
-import borgmatic.hooks.dump
 
 logger = logging.getLogger(__name__)
 
@@ -186,7 +185,7 @@ def run_create(
             'remove_data_source_dumps',
             config,
             repository['path'],
-            borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             borgmatic_runtime_directory,
             global_arguments.dry_run,
         )
@@ -195,7 +194,7 @@ def run_create(
             'dump_data_sources',
             config,
             repository['path'],
-            borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             config_paths,
             borgmatic_runtime_directory,
             source_directories,
@@ -232,7 +231,7 @@ def run_create(
             'remove_data_source_dumps',
             config,
             config_filename,
-            borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             borgmatic_runtime_directory,
             global_arguments.dry_run,
         )

borgmatic/actions/restore.py

@@ -11,8 +11,8 @@ import borgmatic.borg.mount
 import borgmatic.borg.repo_list
 import borgmatic.config.paths
 import borgmatic.config.validate
+import borgmatic.hooks.data_source.dump
 import borgmatic.hooks.dispatch
-import borgmatic.hooks.dump
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +44,8 @@ def get_configured_data_source(
         hooks_to_search = {
             hook_name: value
             for (hook_name, value) in config.items()
-            if hook_name in borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES
+            if hook_name.split('_databases')[0]
+            in borgmatic.hooks.dispatch.get_submodule_names(borgmatic.hooks.data_source)
         }
     else:
         try:
@@ -123,10 +124,10 @@ def restore_single_data_source(
         'make_data_source_dump_patterns',
         config,
         repository['path'],
-        borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+        borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
         borgmatic_runtime_directory,
         data_source['name'],
-    )[hook_name]
+    )[hook_name.split('_databases')[0]]
 
     destination_path = (
         tempfile.mkdtemp(dir=borgmatic_runtime_directory)
@@ -141,7 +142,11 @@ def restore_single_data_source(
             dry_run=global_arguments.dry_run,
             repository=repository['path'],
             archive=archive_name,
-            paths=[borgmatic.hooks.dump.convert_glob_patterns_to_borg_pattern(dump_patterns)],
+            paths=[
+                borgmatic.hooks.data_source.dump.convert_glob_patterns_to_borg_pattern(
+                    dump_patterns
+                )
+            ],
             config=config,
             local_borg_version=local_borg_version,
             global_arguments=global_arguments,
@@ -162,11 +167,11 @@ def restore_single_data_source(
             shutil.rmtree(destination_path, ignore_errors=True)
 
     # Run a single data source restore, consuming the extract stdout (if any).
-    borgmatic.hooks.dispatch.call_hooks(
+    borgmatic.hooks.dispatch.call_hook(
         function_name='restore_data_source_dump',
         config=config,
         log_prefix=repository['path'],
-        hook_names=[hook_name],
+        hook_name=hook_name,
         data_source=data_source,
         dry_run=global_arguments.dry_run,
         extract_process=extract_process,
@@ -206,7 +211,9 @@ def collect_archive_data_source_names(
         global_arguments,
         list_paths=[
             'sh:'
-            + borgmatic.hooks.dump.make_data_source_dump_path(base_directory, '*_databases/*/*')
+            + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
+                base_directory, '*_databases/*/*'
+            )
             for base_directory in (
                 'borgmatic',
                 borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
@@ -354,7 +361,7 @@ def run_restore(
             'remove_data_source_dumps',
             config,
             repository['path'],
-            borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             borgmatic_runtime_directory,
             global_arguments.dry_run,
         )
@@ -451,7 +458,7 @@ def run_restore(
             'remove_data_source_dumps',
             config,
             repository['path'],
-            borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             borgmatic_runtime_directory,
             global_arguments.dry_run,
         )

borgmatic/commands/borgmatic.py

@@ -39,7 +39,8 @@ from borgmatic.borg import umount as borg_umount
 from borgmatic.borg import version as borg_version
 from borgmatic.commands.arguments import parse_arguments
 from borgmatic.config import checks, collect, validate
-from borgmatic.hooks import command, dispatch, monitor
+from borgmatic.hooks import command, dispatch
+from borgmatic.hooks.monitoring import monitor
 from borgmatic.logger import DISABLED, add_custom_log_levels, configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
@@ -103,7 +104,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
                 'initialize_monitor',
                 config,
                 config_filename,
-                monitor.MONITOR_HOOK_NAMES,
+                dispatch.Hook_type.MONITORING,
                 monitoring_log_level,
                 global_arguments.dry_run,
             )
@@ -112,7 +113,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
                 'ping_monitor',
                 config,
                 config_filename,
-                monitor.MONITOR_HOOK_NAMES,
+                dispatch.Hook_type.MONITORING,
                 monitor.State.START,
                 monitoring_log_level,
                 global_arguments.dry_run,
@@ -188,7 +189,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
                 'ping_monitor',
                 config,
                 config_filename,
-                monitor.MONITOR_HOOK_NAMES,
+                dispatch.Hook_type.MONITORING,
                 monitor.State.LOG,
                 monitoring_log_level,
                 global_arguments.dry_run,
@@ -205,7 +206,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
                     'ping_monitor',
                     config,
                     config_filename,
-                    monitor.MONITOR_HOOK_NAMES,
+                    dispatch.Hook_type.MONITORING,
                     monitor.State.FINISH,
                     monitoring_log_level,
                     global_arguments.dry_run,
@@ -214,7 +215,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
                     'destroy_monitor',
                     config,
                     config_filename,
-                    monitor.MONITOR_HOOK_NAMES,
+                    dispatch.Hook_type.MONITORING,
                     monitoring_log_level,
                     global_arguments.dry_run,
                 )
@@ -241,7 +242,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
                 'ping_monitor',
                 config,
                 config_filename,
-                monitor.MONITOR_HOOK_NAMES,
+                dispatch.Hook_type.MONITORING,
                 monitor.State.FAIL,
                 monitoring_log_level,
                 global_arguments.dry_run,
@@ -250,7 +251,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
                 'destroy_monitor',
                 config,
                 config_filename,
-                monitor.MONITOR_HOOK_NAMES,
+                dispatch.Hook_type.MONITORING,
                 monitoring_log_level,
                 global_arguments.dry_run,
             )

borgmatic/hooks/data_source/dump.py

@@ -5,15 +5,7 @@ import shutil
 
 logger = logging.getLogger(__name__)
 
-DATA_SOURCE_HOOK_NAMES = (
-    'bootstrap',
-    'mariadb_databases',
-    'mysql_databases',
-    'mongodb_databases',
-    'postgresql_databases',
-    'sqlite_databases',
-    'zfs',
-)
+IS_A_HOOK = False
 
 
 def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_name):

borgmatic/hooks/data_source/mariadb.py

@@ -9,7 +9,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
-from borgmatic.hooks import dump
+from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
 

borgmatic/hooks/data_source/mongodb.py

@@ -4,7 +4,7 @@ import shlex
 
 import borgmatic.config.paths
 from borgmatic.execute import execute_command, execute_command_with_processes
-from borgmatic.hooks import dump
+from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
 

borgmatic/hooks/data_source/mysql.py

@@ -9,7 +9,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
-from borgmatic.hooks import dump
+from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
 

borgmatic/hooks/data_source/postgresql.py

@@ -11,7 +11,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
-from borgmatic.hooks import dump
+from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
 

borgmatic/hooks/data_source/sqlite.py

@@ -4,7 +4,7 @@ import shlex
 
 import borgmatic.config.paths
 from borgmatic.execute import execute_command, execute_command_with_processes
-from borgmatic.hooks import dump
+from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
 

borgmatic/hooks/dispatch.py

@@ -1,75 +1,70 @@
+import enum
+import importlib
 import logging
+import pkgutil
 
-from borgmatic.hooks import (
-    apprise,
-    bootstrap,
-    cronhub,
-    cronitor,
-    healthchecks,
-    loki,
-    mariadb,
-    mongodb,
-    mysql,
-    ntfy,
-    pagerduty,
-    postgresql,
-    pushover,
-    sqlite,
-    uptimekuma,
-    zabbix,
-    zfs,
-)
+import borgmatic.hooks.data_source
+import borgmatic.hooks.monitoring
 
 logger = logging.getLogger(__name__)
 
-HOOK_NAME_TO_MODULE = {
-    'apprise': apprise,
-    'bootstrap': bootstrap,
-    'cronhub': cronhub,
-    'cronitor': cronitor,
-    'healthchecks': healthchecks,
-    'loki': loki,
-    'mariadb_databases': mariadb,
-    'mongodb_databases': mongodb,
-    'mysql_databases': mysql,
-    'ntfy': ntfy,
-    'pagerduty': pagerduty,
-    'postgresql_databases': postgresql,
-    'pushover': pushover,
-    'sqlite_databases': sqlite,
-    'uptime_kuma': uptimekuma,
-    'zabbix': zabbix,
-    'zfs': zfs,
-}
+
+class Hook_type(enum.Enum):
+    DATA_SOURCE = 'data_source'
+    MONITORING = 'monitoring'
+
+
+def get_submodule_names(parent_module):  # pragma: no cover
+    '''
+    Given a parent module, return the names of its direct submodules as a tuple of strings.
+    '''
+    return tuple(module_info.name for module_info in pkgutil.iter_modules(parent_module.__path__))
 
 
 def call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs):
     '''
     Given a configuration dict and a prefix to use in log entries, call the requested function of
     the Python module corresponding to the given hook name. Supply that call with the configuration
-    for this hook (if any), the log prefix, and any given args and kwargs. Return any return value.
+    for this hook (if any), the log prefix, and any given args and kwargs. Return the return value
+    of that call or None if the module in question is not a hook.
 
     Raise ValueError if the hook name is unknown.
     Raise AttributeError if the function name is not found in the module.
     Raise anything else that the called function raises.
     '''
-    hook_config = config.get(hook_name) or {}
+    hook_config = config.get(hook_name) or config.get(f'{hook_name}_databases') or {}
+    module_name = hook_name.split('_databases')[0]
 
-    try:
-        module = HOOK_NAME_TO_MODULE[hook_name]
-    except KeyError:
+    # Probe for a data source or monitoring hook module corresponding to the hook name.
+    for parent_module in (borgmatic.hooks.data_source, borgmatic.hooks.monitoring):
+        if module_name not in get_submodule_names(parent_module):
+            continue
+
+        module = importlib.import_module(f'{parent_module.__name__}.{module_name}')
+
+        # If this module is explicitly flagged as not a hook, bail.
+        if not getattr(module, 'IS_A_HOOK', True):
+            return None
+
+        break
+    else:
         raise ValueError(f'Unknown hook name: {hook_name}')
 
     logger.debug(f'{log_prefix}: Calling {hook_name} hook function {function_name}')
+
     return getattr(module, function_name)(hook_config, config, log_prefix, *args, **kwargs)
 
 
-def call_hooks(function_name, config, log_prefix, hook_names, *args, **kwargs):
+def call_hooks(function_name, config, log_prefix, hook_type, *args, **kwargs):
     '''
     Given a configuration dict and a prefix to use in log entries, call the requested function of
-    the Python module corresponding to each given hook name. Supply each call with the configuration
-    for that hook, the log prefix, and any given args and kwargs. Collect any return values into a
-    dict from hook name to return value.
+    the Python module corresponding to each hook of the given hook type (either "data_source" or
+    "monitoring"). Supply each call with the configuration for that hook, the log prefix, and any
+    given args and kwargs.
+
+    Collect any return values into a dict from module name to return value. Note that the module
+    name is the name of the hook module itself, which might be different from the hook configuration
+    option (e.g. "postgresql" for the former vs. "postgresql_databases" for the latter).
 
     If the hook name is not present in the hooks configuration, then don't call the function for it
     and omit it from the return values.
@@ -80,22 +75,26 @@ def call_hooks(function_name, config, log_prefix, hook_names, *args, **kwargs):
     '''
     return {
         hook_name: call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs)
-        for hook_name in hook_names
-        if hook_name in config
+        for hook_name in get_submodule_names(
+            importlib.import_module(f'borgmatic.hooks.{hook_type.value}')
+        )
+        if hook_name in config or f'{hook_name}_databases' in config
     }
 
 
-def call_hooks_even_if_unconfigured(function_name, config, log_prefix, hook_names, *args, **kwargs):
+def call_hooks_even_if_unconfigured(function_name, config, log_prefix, hook_type, *args, **kwargs):
     '''
     Given a configuration dict and a prefix to use in log entries, call the requested function of
-    the Python module corresponding to each given hook name. Supply each call with the configuration
-    for that hook, the log prefix, and any given args and kwargs. Collect any return values into a
-    dict from hook name to return value.
+    the Python module corresponding to each hook of the given hook type (either "data_source" or
+    "monitoring"). Supply each call with the configuration for that hook, the log prefix, and any
+    given args and kwargs. Collect any return values into a dict from hook name to return value.
 
     Raise AttributeError if the function name is not found in the module.
     Raise anything else that a called function raises. An error stops calls to subsequent functions.
     '''
     return {
         hook_name: call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs)
-        for hook_name in hook_names
+        for hook_name in get_submodule_names(
+            importlib.import_module(f'borgmatic.hooks.{hook_type.value}')
+        )
     }

borgmatic/hooks/monitor.py

@@ -1,21 +0,0 @@
-from enum import Enum
-
-MONITOR_HOOK_NAMES = (
-    'apprise',
-    'cronhub',
-    'cronitor',
-    'healthchecks',
-    'loki',
-    'ntfy',
-    'pagerduty',
-    'pushover',
-    'uptime_kuma',
-    'zabbix',
-)
-
-
-class State(Enum):
-    START = 1
-    FINISH = 2
-    FAIL = 3
-    LOG = 4

borgmatic/hooks/monitoring/apprise.py

@@ -1,8 +1,8 @@
 import logging
 import operator
 
-import borgmatic.hooks.logs
-import borgmatic.hooks.monitor
+import borgmatic.hooks.monitoring.logs
+import borgmatic.hooks.monitoring.monitor
 
 logger = logging.getLogger(__name__)
 
@@ -22,12 +22,12 @@ def initialize_monitor(hook_config, config, config_filename, monitoring_log_leve
 
     logs_size_limit = max(
         hook_config.get('logs_size_limit', DEFAULT_LOGS_SIZE_LIMIT_BYTES)
-        - len(borgmatic.hooks.logs.PAYLOAD_TRUNCATION_INDICATOR),
+        - len(borgmatic.hooks.monitoring.logs.PAYLOAD_TRUNCATION_INDICATOR),
         0,
     )
 
-    borgmatic.hooks.logs.add_handler(
-        borgmatic.hooks.logs.Forgetful_buffering_handler(
+    borgmatic.hooks.monitoring.logs.add_handler(
+        borgmatic.hooks.monitoring.logs.Forgetful_buffering_handler(
             HANDLER_IDENTIFIER, logs_size_limit, monitoring_log_level
         )
     )
@@ -82,11 +82,13 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     body = state_config.get('body')
 
     if state in (
-        borgmatic.hooks.monitor.State.FINISH,
-        borgmatic.hooks.monitor.State.FAIL,
-        borgmatic.hooks.monitor.State.LOG,
+        borgmatic.hooks.monitoring.monitor.State.FINISH,
+        borgmatic.hooks.monitoring.monitor.State.FAIL,
+        borgmatic.hooks.monitoring.monitor.State.LOG,
     ):
-        formatted_logs = borgmatic.hooks.logs.format_buffered_logs_for_payload(HANDLER_IDENTIFIER)
+        formatted_logs = borgmatic.hooks.monitoring.logs.format_buffered_logs_for_payload(
+            HANDLER_IDENTIFIER
+        )
         if formatted_logs:
             body += f'\n\n{formatted_logs}'
 
@@ -106,4 +108,4 @@ def destroy_monitor(hook_config, config, config_filename, monitoring_log_level,
     Remove the monitor handler that was added to the root logger. This prevents the handler from
     getting reused by other instances of this monitor.
     '''
-    borgmatic.hooks.logs.remove_handler(HANDLER_IDENTIFIER)
+    borgmatic.hooks.monitoring.logs.remove_handler(HANDLER_IDENTIFIER)

borgmatic/hooks/monitoring/cronhub.py

@@ -2,7 +2,7 @@ import logging
 
 import requests
 
-from borgmatic.hooks import monitor
+from borgmatic.hooks.monitoring import monitor
 
 logger = logging.getLogger(__name__)
 

borgmatic/hooks/monitoring/cronitor.py

@@ -2,7 +2,7 @@ import logging
 
 import requests
 
-from borgmatic.hooks import monitor
+from borgmatic.hooks.monitoring import monitor
 
 logger = logging.getLogger(__name__)
 

borgmatic/hooks/monitoring/healthchecks.py

@@ -3,8 +3,8 @@ import re
 
 import requests
 
-import borgmatic.hooks.logs
-from borgmatic.hooks import monitor
+import borgmatic.hooks.monitoring.logs
+from borgmatic.hooks.monitoring import monitor
 
 logger = logging.getLogger(__name__)
 
@@ -30,12 +30,12 @@ def initialize_monitor(hook_config, config, config_filename, monitoring_log_leve
 
     ping_body_limit = max(
         hook_config.get('ping_body_limit', DEFAULT_PING_BODY_LIMIT_BYTES)
-        - len(borgmatic.hooks.logs.PAYLOAD_TRUNCATION_INDICATOR),
+        - len(borgmatic.hooks.monitoring.logs.PAYLOAD_TRUNCATION_INDICATOR),
         0,
     )
 
-    borgmatic.hooks.logs.add_handler(
-        borgmatic.hooks.logs.Forgetful_buffering_handler(
+    borgmatic.hooks.monitoring.logs.add_handler(
+        borgmatic.hooks.monitoring.logs.Forgetful_buffering_handler(
             HANDLER_IDENTIFIER, ping_body_limit, monitoring_log_level
         )
     )
@@ -78,7 +78,9 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     logger.debug(f'{config_filename}: Using Healthchecks ping URL {ping_url}')
 
     if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG):
-        payload = borgmatic.hooks.logs.format_buffered_logs_for_payload(HANDLER_IDENTIFIER)
+        payload = borgmatic.hooks.monitoring.logs.format_buffered_logs_for_payload(
+            HANDLER_IDENTIFIER
+        )
     else:
         payload = ''
 
@@ -99,4 +101,4 @@ def destroy_monitor(hook_config, config, config_filename, monitoring_log_level,
     Remove the monitor handler that was added to the root logger. This prevents the handler from
     getting reused by other instances of this monitor.
     '''
-    borgmatic.hooks.logs.remove_handler(HANDLER_IDENTIFIER)
+    borgmatic.hooks.monitoring.logs.remove_handler(HANDLER_IDENTIFIER)

borgmatic/hooks/monitoring/logs.py

@@ -1,5 +1,6 @@
 import logging
 
+IS_A_HOOK = False
 PAYLOAD_TRUNCATION_INDICATOR = '...\n'
 
 

borgmatic/hooks/monitoring/loki.py

@@ -6,7 +6,7 @@ import time
 
 import requests
 
-from borgmatic.hooks import monitor
+from borgmatic.hooks.monitoring import monitor
 
 logger = logging.getLogger(__name__)
 

borgmatic/hooks/monitoring/monitor.py

@@ -0,0 +1,10 @@
+import enum
+
+IS_A_HOOK = False
+
+
+class State(enum.Enum):
+    START = 1
+    FINISH = 2
+    FAIL = 3
+    LOG = 4

borgmatic/hooks/monitoring/pagerduty.py

@@ -5,7 +5,7 @@ import platform
 
 import requests
 
-from borgmatic.hooks import monitor
+from borgmatic.hooks.monitoring import monitor
 
 logger = logging.getLogger(__name__)
 

docs/reference/source-code.md

@@ -16,8 +16,10 @@ you get started. Starting at the top level, we have:
    * [actions](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/actions): borgmatic-specific logic for running each action (create, list, check, etc.).
    * [borg](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/borg): Lower-level code that's responsible for interacting with Borg to run each action.
    * [commands](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/commands): Looking to add a new flag or action? Start here. This contains borgmatic's entry point, argument parsing, and shell completion. 
-   * [config](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/config): Code responsible for loading, normalizing, and validating borgmatic's configuration.
-   * [hooks](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/hooks): Looking to add a new database or monitoring integration? Start here.
+   * [config](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/config): Code responsible for loading, normalizing, and validating borgmatic's configuration. Interested in adding a new configuration option? Check out `schema.yaml` here.
+   * [hooks](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/hooks): Looking to add a new database, filesystem, or monitoring integration? Start here.
+     * [data_source](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/hooks): Database and filesystem hooks—anything that produces data or files to go into a backup archive.
+     * [monitoring](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/hooks): Monitoring hooks—integrations with third-party or self-hosted monitoring services.
  * [docs](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/docs): How-to and reference documentation, including the document you're reading now.
  * [sample](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/sample): Example configurations for cron and systemd.
  * [scripts](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/scripts): Dev-facing scripts for things like building documentation and running end-to-end tests.

tests/integration/hooks/monitoring/test_apprise.py

@@ -2,14 +2,14 @@ import logging
 
 from flexmock import flexmock
 
-from borgmatic.hooks import apprise as module
+from borgmatic.hooks.monitoring import apprise as module
 
 
 def test_destroy_monitor_removes_apprise_handler():
     logger = logging.getLogger()
     original_handlers = list(logger.handlers)
-    module.borgmatic.hooks.logs.add_handler(
-        module.borgmatic.hooks.logs.Forgetful_buffering_handler(
+    module.borgmatic.hooks.monitoring.logs.add_handler(
+        module.borgmatic.hooks.monitoring.logs.Forgetful_buffering_handler(
             identifier=module.HANDLER_IDENTIFIER, byte_capacity=100, log_level=1
         )
     )

tests/integration/hooks/monitoring/test_healthchecks.py

@@ -2,14 +2,14 @@ import logging
 
 from flexmock import flexmock
 
-from borgmatic.hooks import healthchecks as module
+from borgmatic.hooks.monitoring import healthchecks as module
 
 
 def test_destroy_monitor_removes_healthchecks_handler():
     logger = logging.getLogger()
     original_handlers = list(logger.handlers)
-    module.borgmatic.hooks.logs.add_handler(
-        module.borgmatic.hooks.logs.Forgetful_buffering_handler(
+    module.borgmatic.hooks.monitoring.logs.add_handler(
+        module.borgmatic.hooks.monitoring.logs.Forgetful_buffering_handler(
             identifier=module.HANDLER_IDENTIFIER, byte_capacity=100, log_level=1
         )
     )

tests/integration/hooks/monitoring/test_loki.py

@@ -3,7 +3,7 @@ import platform
 
 from flexmock import flexmock
 
-from borgmatic.hooks import loki as module
+from borgmatic.hooks.monitoring import loki as module
 
 
 def test_initialize_monitor_replaces_labels():

tests/unit/actions/test_restore.py

@@ -90,7 +90,7 @@ def test_restore_single_data_source_extracts_and_restores_single_file_dump():
         'make_data_source_dump_patterns', object, object, object, object, object
     ).and_return({'postgresql': flexmock()})
     flexmock(module.tempfile).should_receive('mkdtemp').never()
-    flexmock(module.borgmatic.hooks.dump).should_receive(
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
         'convert_glob_patterns_to_borg_pattern'
     ).and_return(flexmock())
     flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
@@ -98,11 +98,11 @@ def test_restore_single_data_source_extracts_and_restores_single_file_dump():
     ).once()
     flexmock(module).should_receive('strip_path_prefix_from_extracted_dump_destination').never()
     flexmock(module.shutil).should_receive('rmtree').never()
-    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args(
+    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args(
         function_name='restore_data_source_dump',
         config=object,
         log_prefix=object,
-        hook_names=object,
+        hook_name=object,
         data_source=object,
         dry_run=object,
         extract_process=object,
@@ -132,7 +132,7 @@ def test_restore_single_data_source_extracts_and_restores_directory_dump():
     flexmock(module.tempfile).should_receive('mkdtemp').once().and_return(
         '/run/user/0/borgmatic/tmp1234'
     )
-    flexmock(module.borgmatic.hooks.dump).should_receive(
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
         'convert_glob_patterns_to_borg_pattern'
     ).and_return(flexmock())
     flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
@@ -140,11 +140,11 @@ def test_restore_single_data_source_extracts_and_restores_directory_dump():
     ).once()
     flexmock(module).should_receive('strip_path_prefix_from_extracted_dump_destination').once()
     flexmock(module.shutil).should_receive('rmtree').once()
-    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args(
+    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args(
         function_name='restore_data_source_dump',
         config=object,
         log_prefix=object,
-        hook_names=object,
+        hook_name=object,
         data_source=object,
         dry_run=object,
         extract_process=object,
@@ -174,7 +174,7 @@ def test_restore_single_data_source_with_directory_dump_error_cleans_up_temporar
     flexmock(module.tempfile).should_receive('mkdtemp').once().and_return(
         '/run/user/0/borgmatic/tmp1234'
     )
-    flexmock(module.borgmatic.hooks.dump).should_receive(
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
         'convert_glob_patterns_to_borg_pattern'
     ).and_return(flexmock())
     flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_raise(
@@ -182,11 +182,11 @@ def test_restore_single_data_source_with_directory_dump_error_cleans_up_temporar
     ).once()
     flexmock(module).should_receive('strip_path_prefix_from_extracted_dump_destination').never()
     flexmock(module.shutil).should_receive('rmtree').once()
-    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args(
+    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args(
         function_name='restore_data_source_dump',
         config=object,
         log_prefix=object,
-        hook_names=object,
+        hook_name=object,
         data_source=object,
         dry_run=object,
         extract_process=object,
@@ -215,7 +215,7 @@ def test_restore_single_data_source_with_directory_dump_and_dry_run_skips_direct
         'make_data_source_dump_patterns', object, object, object, object, object
     ).and_return({'postgresql': flexmock()})
     flexmock(module.tempfile).should_receive('mkdtemp').once().and_return('/run/borgmatic/tmp1234')
-    flexmock(module.borgmatic.hooks.dump).should_receive(
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
         'convert_glob_patterns_to_borg_pattern'
     ).and_return(flexmock())
     flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
@@ -223,11 +223,11 @@ def test_restore_single_data_source_with_directory_dump_and_dry_run_skips_direct
     ).once()
     flexmock(module).should_receive('strip_path_prefix_from_extracted_dump_destination').never()
     flexmock(module.shutil).should_receive('rmtree').never()
-    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args(
+    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args(
         function_name='restore_data_source_dump',
         config=object,
         log_prefix=object,
-        hook_names=object,
+        hook_name=object,
         data_source=object,
         dry_run=object,
         extract_process=object,
@@ -254,9 +254,9 @@ def test_collect_archive_data_source_names_parses_archive_paths():
     flexmock(module.borgmatic.config.paths).should_receive(
         'get_borgmatic_source_directory'
     ).and_return('/root/.borgmatic')
-    flexmock(module.borgmatic.hooks.dump).should_receive('make_data_source_dump_path').and_return(
-        ''
-    )
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'make_data_source_dump_path'
+    ).and_return('')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
         [
             'borgmatic/postgresql_databases/localhost/foo',
@@ -286,9 +286,9 @@ def test_collect_archive_data_source_names_parses_archive_paths_with_different_b
     flexmock(module.borgmatic.config.paths).should_receive(
         'get_borgmatic_source_directory'
     ).and_return('/root/.borgmatic')
-    flexmock(module.borgmatic.hooks.dump).should_receive('make_data_source_dump_path').and_return(
-        ''
-    )
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'make_data_source_dump_path'
+    ).and_return('')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
         [
             'borgmatic/postgresql_databases/localhost/foo',
@@ -319,9 +319,9 @@ def test_collect_archive_data_source_names_parses_directory_format_archive_paths
     flexmock(module.borgmatic.config.paths).should_receive(
         'get_borgmatic_source_directory'
     ).and_return('/root/.borgmatic')
-    flexmock(module.borgmatic.hooks.dump).should_receive('make_data_source_dump_path').and_return(
-        ''
-    )
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'make_data_source_dump_path'
+    ).and_return('')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
         [
             'borgmatic/postgresql_databases/localhost/foo/table1',
@@ -349,9 +349,9 @@ def test_collect_archive_data_source_names_skips_bad_archive_paths():
     flexmock(module.borgmatic.config.paths).should_receive(
         'get_borgmatic_source_directory'
     ).and_return('/root/.borgmatic')
-    flexmock(module.borgmatic.hooks.dump).should_receive('make_data_source_dump_path').and_return(
-        ''
-    )
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'make_data_source_dump_path'
+    ).and_return('')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
         [
             'borgmatic/postgresql_databases/localhost/foo',

tests/unit/hooks/data_source/test_bootstrap.py

@@ -2,7 +2,7 @@ import sys
 
 from flexmock import flexmock
 
-from borgmatic.hooks import bootstrap as module
+from borgmatic.hooks.data_source import bootstrap as module
 
 
 def test_dump_data_sources_creates_manifest_file():

tests/unit/hooks/data_source/test_dump.py

@@ -1,7 +1,7 @@
 import pytest
 from flexmock import flexmock
 
-from borgmatic.hooks import dump as module
+from borgmatic.hooks.data_source import dump as module
 
 
 def test_make_data_source_dump_path_joins_arguments():

tests/unit/hooks/data_source/test_mariadb.py

@@ -3,7 +3,7 @@ import logging
 import pytest
 from flexmock import flexmock
 
-from borgmatic.hooks import mariadb as module
+from borgmatic.hooks.data_source import mariadb as module
 
 
 def test_database_names_to_dump_passes_through_name():

tests/unit/hooks/data_source/test_mongodb.py

@@ -2,7 +2,7 @@ import logging
 
 from flexmock import flexmock
 
-from borgmatic.hooks import mongodb as module
+from borgmatic.hooks.data_source import mongodb as module
 
 
 def test_use_streaming_true_for_any_non_directory_format_databases():

tests/unit/hooks/data_source/test_mysql.py

@@ -3,7 +3,7 @@ import logging
 import pytest
 from flexmock import flexmock
 
-from borgmatic.hooks import mysql as module
+from borgmatic.hooks.data_source import mysql as module
 
 
 def test_database_names_to_dump_passes_through_name():

tests/unit/hooks/data_source/test_postgresql.py

@@ -3,7 +3,7 @@ import logging
 import pytest
 from flexmock import flexmock
 
-from borgmatic.hooks import postgresql as module
+from borgmatic.hooks.data_source import postgresql as module
 
 
 def test_make_extra_environment_maps_options_to_environment():

tests/unit/hooks/data_source/test_sqlite.py

@@ -2,7 +2,7 @@ import logging
 
 from flexmock import flexmock
 
-from borgmatic.hooks import sqlite as module
+from borgmatic.hooks.data_source import sqlite as module
 
 
 def test_use_streaming_true_for_any_databases():

tests/unit/hooks/data_source/test_zfs.py

@@ -2,7 +2,7 @@ import pytest
 from flexmock import flexmock
 
 import borgmatic.execute
-from borgmatic.hooks import zfs as module
+from borgmatic.hooks.data_source import zfs as module
 
 
 def test_get_datasets_to_backup_filters_datasets_by_source_directories():

tests/unit/hooks/monitoring/test_apprise.py

@@ -2,8 +2,8 @@ import apprise
 from apprise import NotifyFormat, NotifyType
 from flexmock import flexmock
 
-import borgmatic.hooks.monitor
-from borgmatic.hooks import apprise as module
+import borgmatic.hooks.monitoring.monitor
+from borgmatic.hooks.monitoring import apprise as module
 
 TOPIC = 'borgmatic-unit-testing'
 
@@ -18,7 +18,7 @@ def mock_apprise():
 
 
 def test_initialize_monitor_with_send_logs_false_does_not_add_handler():
-    flexmock(module.borgmatic.hooks.logs).should_receive('add_handler').never()
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive('add_handler').never()
 
     module.initialize_monitor(
         hook_config={'send_logs': False},
@@ -31,12 +31,14 @@ def test_initialize_monitor_with_send_logs_false_does_not_add_handler():
 
 def test_initialize_monitor_with_send_logs_true_adds_handler_with_default_log_size_limit():
     truncation_indicator_length = 4
-    flexmock(module.borgmatic.hooks.logs).should_receive('Forgetful_buffering_handler').with_args(
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+        'Forgetful_buffering_handler'
+    ).with_args(
         module.HANDLER_IDENTIFIER,
         module.DEFAULT_LOGS_SIZE_LIMIT_BYTES - truncation_indicator_length,
         1,
     ).once()
-    flexmock(module.borgmatic.hooks.logs).should_receive('add_handler').once()
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive('add_handler').once()
 
     module.initialize_monitor(
         hook_config={'send_logs': True},
@@ -49,12 +51,14 @@ def test_initialize_monitor_with_send_logs_true_adds_handler_with_default_log_si
 
 def test_initialize_monitor_without_send_logs_adds_handler_with_default_log_size_limit():
     truncation_indicator_length = 4
-    flexmock(module.borgmatic.hooks.logs).should_receive('Forgetful_buffering_handler').with_args(
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+        'Forgetful_buffering_handler'
+    ).with_args(
         module.HANDLER_IDENTIFIER,
         module.DEFAULT_LOGS_SIZE_LIMIT_BYTES - truncation_indicator_length,
         1,
     ).once()
-    flexmock(module.borgmatic.hooks.logs).should_receive('add_handler').once()
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive('add_handler').once()
 
     module.initialize_monitor(
         hook_config={},
@@ -66,8 +70,8 @@ def test_initialize_monitor_without_send_logs_adds_handler_with_default_log_size
 
 
 def test_ping_monitor_respects_dry_run():
-    flexmock(module.borgmatic.hooks.logs).should_receive('get_handler')
-    flexmock(module.borgmatic.hooks.logs).should_receive(
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive('get_handler')
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
         'format_buffered_logs_for_payload'
     ).and_return('loggy log')
     mock_apprise().should_receive('notify').never()
@@ -76,30 +80,32 @@ def test_ping_monitor_respects_dry_run():
         {'services': [{'url': f'ntfys://{TOPIC}', 'label': 'ntfys'}]},
         {},
         'config.yaml',
-        borgmatic.hooks.monitor.State.FAIL,
+        borgmatic.hooks.monitoring.monitor.State.FAIL,
         monitoring_log_level=1,
         dry_run=True,
     )
 
 
 def test_ping_monitor_with_no_states_does_not_notify():
-    flexmock(module.borgmatic.hooks.logs).should_receive('get_handler').never()
-    flexmock(module.borgmatic.hooks.logs).should_receive('format_buffered_logs_for_payload').never()
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive('get_handler').never()
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+        'format_buffered_logs_for_payload'
+    ).never()
     mock_apprise().should_receive('notify').never()
 
     module.ping_monitor(
         {'services': [{'url': f'ntfys://{TOPIC}', 'label': 'ntfys'}], 'states': []},
         {},
         'config.yaml',
-        borgmatic.hooks.monitor.State.FAIL,
+        borgmatic.hooks.monitoring.monitor.State.FAIL,
         monitoring_log_level=1,
         dry_run=True,
     )
 
 
 def test_ping_monitor_notifies_fail_by_default():
-    flexmock(module.borgmatic.hooks.logs).should_receive('get_handler')
-    flexmock(module.borgmatic.hooks.logs).should_receive(
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive('get_handler')
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
         'format_buffered_logs_for_payload'
     ).and_return('')
     mock_apprise().should_receive('notify').with_args(
@@ -109,7 +115,7 @@ def test_ping_monitor_notifies_fail_by_default():
         notify_type=NotifyType.FAILURE,
     ).once()
 
-    for state in borgmatic.hooks.monitor.State:
+    for state in borgmatic.hooks.monitoring.monitor.State:
         module.ping_monitor(
             {'services': [{'url': f'ntfys://{TOPIC}', 'label': 'ntfys'}]},
             {},
@@ -121,8 +127,8 @@ def test_ping_monitor_notifies_fail_by_default():
 
 
 def test_ping_monitor_with_logs_appends_logs_to_body():
-    flexmock(module.borgmatic.hooks.logs).should_receive('get_handler')
-    flexmock(module.borgmatic.hooks.logs).should_receive(
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive('get_handler')
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
         'format_buffered_logs_for_payload'
     ).and_return('loggy log')
     mock_apprise().should_receive('notify').with_args(
@@ -132,7 +138,7 @@ def test_ping_monitor_with_logs_appends_logs_to_body():
         notify_type=NotifyType.FAILURE,
     ).once()
 
-    for state in borgmatic.hooks.monitor.State:
+    for state in borgmatic.hooks.monitoring.monitor.State:
         module.ping_monitor(
             {'services': [{'url': f'ntfys://{TOPIC}', 'label': 'ntfys'}]},
             {},
@@ -144,8 +150,8 @@ def test_ping_monitor_with_logs_appends_logs_to_body():
 
 
 def test_ping_monitor_with_finish_default_config_notifies():
-    flexmock(module.borgmatic.hooks.logs).should_receive('get_handler')
-    flexmock(module.borgmatic.hooks.logs).should_receive(
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive('get_handler')
+    flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
         'format_buffered_logs_for_payload'
     ).and_return('')
     mock_apprise().should_receive('notify').with_args(
@@ -159,15 +165,17 @@ def test_ping_monitor_with_finish_default_config_notifies():
         {'services': [{'url': f'ntfys://{TOPIC}', 'label': 'ntfys'}], 'states': ['finish']},
         {},
         'config.yaml',