Change monitoring hooks to specify the ping URL / integration key as a named option.

NEWS

@@ -8,6 +8,9 @@
    directory or from the directory containing the file doing the including. Previously, only the
    working directory was used.
  * Add a randomized delay to the sample systemd timer to spread out the load on a server.
+ * Change the configuration format for borgmatic monitoring hooks (Healthchecks, Cronitor,
+   PagerDuty, and Cronhub) to specify the ping URL / integration key as a named option. The intent
+   is to support additional options in the future. This change is backwards-compatible.
  * Add emojis to documentation table of contents to make it easier to find particular how-to and
    reference guides at a glance.
 

borgmatic/config/normalize.py

@@ -3,8 +3,24 @@ def normalize(config):
     Given a configuration dict, apply particular hard-coded rules to normalize its contents to
     adhere to the configuration schema.
     '''
+    # Upgrade exclude_if_present from a string to a list.
     exclude_if_present = config.get('location', {}).get('exclude_if_present')
-
-    # "Upgrade" exclude_if_present from a string to a list.
     if isinstance(exclude_if_present, str):
         config['location']['exclude_if_present'] = [exclude_if_present]
+
+    # Upgrade various monitoring hooks from a string to a dict.
+    healthchecks = config.get('hooks', {}).get('healthchecks')
+    if isinstance(healthchecks, str):
+        config['hooks']['healthchecks'] = {'ping_url': healthchecks}
+
+    cronitor = config.get('hooks', {}).get('cronitor')
+    if isinstance(cronitor, str):
+        config['hooks']['cronitor'] = {'ping_url': cronitor}
+
+    pagerduty = config.get('hooks', {}).get('pagerduty')
+    if isinstance(pagerduty, str):
+        config['hooks']['pagerduty'] = {'integration_key': pagerduty}
+
+    cronhub = config.get('hooks', {}).get('cronhub')
+    if isinstance(cronhub, str):
+        config['hooks']['cronhub'] = {'ping_url': cronhub}

borgmatic/config/schema.yaml

@@ -882,41 +882,69 @@ properties:
                     https://docs.mongodb.com/database-tools/mongorestore/ for
                     details.
             healthchecks:
-                type: string
+                type: object
+                required: ['ping_url']
+                additionalProperties: false
+                properties:
+                    ping_url:
+                        type: string
+                        description: |
+                            Healthchecks ping URL or UUID to notify when a
+                            backup begins, ends, or errors.
+                        example: https://hc-ping.com/your-uuid-here
                 description: |
-                    Healthchecks ping URL or UUID to notify when a backup
+                    Configuration for a monitoring integration with
-                    begins, ends, or errors. Create an account at
+                    Healthchecks. Create an account at https://healthchecks.io
-                    https://healthchecks.io if you'd like to use this service.
+                    (or self-host Healthchecks) if you'd like to use this
-                    See borgmatic monitoring documentation for details.
-                example:
-                    https://hc-ping.com/your-uuid-here
-            cronitor:
-                type: string
-                description: |
-                    Cronitor ping URL to notify when a backup begins, ends, or
-                    errors. Create an account at https://cronitor.io if you'd
-                    like to use this service. See borgmatic monitoring
-                    documentation for details.
-                example:
-                    https://cronitor.link/d3x0c1
-            pagerduty:
-                type: string
-                description: |
-                    PagerDuty integration key used to notify PagerDuty when a
-                    backup errors. Create an account at
-                    https://www.pagerduty.com/ if you'd like to use this
                     service. See borgmatic monitoring documentation for details.
-                example:
+            cronitor:
-                    a177cad45bd374409f78906a810a3074
+                type: object
-            cronhub:
+                required: ['ping_url']
-                type: string
+                additionalProperties: false
+                properties:
+                    ping_url:
+                        type: string
+                        description: |
+                            Cronitor ping URL to notify when a backup begins,
+                            ends, or errors.
+                        example: https://cronitor.link/d3x0c1
                 description: |
-                    Cronhub ping URL to notify when a backup begins, ends, or
+                    Configuration for a monitoring integration with Cronitor.
-                    errors. Create an account at https://cronhub.io if you'd
+                    Create an account at https://cronitor.io if you'd
                     like to use this service. See borgmatic monitoring
                     documentation for details.
-                example:
+            pagerduty:
-                    https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01
+                type: object
+                required: ['integration_key']
+                additionalProperties: false
+                properties:
+                    integration_key:
+                        type: string
+                        description: |
+                            PagerDuty integration key used to notify PagerDuty
+                            when a backup errors.
+                        example: a177cad45bd374409f78906a810a3074
+                description: |
+                    Configuration for a monitoring integration with PagerDuty.
+                    Create an account at https://www.pagerduty.com/ if you'd
+                    like to use this service. See borgmatic monitoring
+                    documentation for details.
+            cronhub:
+                type: object
+                required: ['ping_url']
+                additionalProperties: false
+                properties:
+                    ping_url:
+                        type: string
+                        description: |
+                            Cronhub ping URL to notify when a backup begins,
+                            ends, or errors.
+                        example: https://cronhub.io/ping/1f5e3410-254c-5587
+                description: |
+                    Configuration for a monitoring integration with Crunhub.
+                    Create an account at https://cronhub.io if you'd like to
+                    use this service. See borgmatic monitoring documentation
+                    for details.
             umask:
                 type: integer
                 description: |

borgmatic/hooks/cronhub.py

@@ -22,14 +22,18 @@ def initialize_monitor(
     pass
 
 
-def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
+def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
     '''
-    Ping the given Cronhub URL, modified with the monitor.State. Use the given configuration
+    Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration
     filename in any log entries. If this is a dry run, then don't actually ping anything.
     '''
     dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
     formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state])
-    ping_url = ping_url.replace('/start/', formatted_state).replace('/ping/', formatted_state)
+    ping_url = (
+        hook_config['ping_url']
+        .replace('/start/', formatted_state)
+        .replace('/ping/', formatted_state)
+    )
 
     logger.info(
         '{}: Pinging Cronhub {}{}'.format(config_filename, state.name.lower(), dry_run_label)

borgmatic/hooks/cronitor.py

@@ -22,13 +22,13 @@ def initialize_monitor(
     pass
 
 
-def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
+def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
     '''
-    Ping the given Cronitor URL, modified with the monitor.State. Use the given configuration
+    Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration
     filename in any log entries. If this is a dry run, then don't actually ping anything.
     '''
     dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
-    ping_url = '{}/{}'.format(ping_url, MONITOR_STATE_TO_CRONITOR[state])
+    ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state])
 
     logger.info(
         '{}: Pinging Cronitor {}{}'.format(config_filename, state.name.lower(), dry_run_label)

borgmatic/hooks/healthchecks.py

@@ -66,7 +66,7 @@ def format_buffered_logs_for_payload():
 
 
 def initialize_monitor(
-    ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
+    hook_config, config_filename, monitoring_log_level, dry_run
 ):  # pragma: no cover
     '''
     Add a handler to the root logger that stores in memory the most recent logs emitted. That
@@ -77,16 +77,16 @@ def initialize_monitor(
     )
 
 
-def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level, dry_run):
+def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
     '''
-    Ping the given Healthchecks URL or UUID, modified with the monitor.State. Use the given
+    Ping the configured Healthchecks URL or UUID, modified with the monitor.State. Use the given
     configuration filename in any log entries, and log to Healthchecks with the giving log level.
     If this is a dry run, then don't actually ping anything.
     '''
     ping_url = (
-        ping_url_or_uuid
+        hook_config['ping_url']
-        if ping_url_or_uuid.startswith('http')
+        if hook_config['ping_url'].startswith('http')
-        else 'https://hc-ping.com/{}'.format(ping_url_or_uuid)
+        else 'https://hc-ping.com/{}'.format(hook_config['ping_url'])
     )
     dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
 
@@ -109,7 +109,7 @@ def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level,
         requests.post(ping_url, data=payload.encode('utf-8'))
 
 
-def destroy_monitor(ping_url_or_uuid, config_filename, monitoring_log_level, dry_run):
+def destroy_monitor(hook_config, config_filename, monitoring_log_level, dry_run):
     '''
     Remove the monitor handler that was added to the root logger. This prevents the handler from
     getting reused by other instances of this monitor.

borgmatic/hooks/pagerduty.py

@@ -21,10 +21,10 @@ def initialize_monitor(
     pass
 
 
-def ping_monitor(integration_key, config_filename, state, monitoring_log_level, dry_run):
+def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
     '''
-    If this is an error state, create a PagerDuty event with the given integration key. Use the
+    If this is an error state, create a PagerDuty event with the configured integration key. Use
-    given configuration filename in any log entries. If this is a dry run, then don't actually
+    the given configuration filename in any log entries. If this is a dry run, then don't actually
     create an event.
     '''
     if state != monitor.State.FAIL:
@@ -47,7 +47,7 @@ def ping_monitor(integration_key, config_filename, state, monitoring_log_level,
     )
     payload = json.dumps(
         {
-            'routing_key': integration_key,
+            'routing_key': hook_config['integration_key'],
             'event_action': 'trigger',
             'payload': {
                 'summary': 'backup failed on {}'.format(hostname),

docs/how-to/monitor-your-backups.md

@@ -136,7 +136,8 @@ URL" for your project. Here's an example:
 
 ```yaml
 hooks:
-    healthchecks: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a
+    healthchecks:
+        ping_url: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a
 ```
 
 With this hook in place, borgmatic pings your Healthchecks project when a
@@ -176,7 +177,8 @@ API URL" for your monitor. Here's an example:
 
 ```yaml
 hooks:
-    cronitor: https://cronitor.link/d3x0c1
+    cronitor:
+        ping_url: https://cronitor.link/d3x0c1
 ```
 
 With this hook in place, borgmatic pings your Cronitor monitor when a backup
@@ -204,7 +206,8 @@ URL" for your monitor. Here's an example:
 
 ```yaml
 hooks:
-    cronhub: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
+    cronhub:
+        ping_url: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
 ```
 
 With this hook in place, borgmatic pings your Cronhub monitor when a backup
@@ -246,7 +249,8 @@ Here's an example:
 
 ```yaml
 hooks:
-    pagerduty: a177cad45bd374409f78906a810a3074
+    pagerduty:
+        integration_key: a177cad45bd374409f78906a810a3074
 ```
 
 With this hook in place, borgmatic creates a PagerDuty event for your service

tests/unit/config/test_normalize.py

@@ -19,6 +19,22 @@ from borgmatic.config import normalize as module
             {'location': {'source_directories': ['foo', 'bar']}},
         ),
         ({'storage': {'compression': 'yes_please'}}, {'storage': {'compression': 'yes_please'}}),
+        (
+            {'hooks': {'healthchecks': 'https://example.com'}},
+            {'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}},
+        ),
+        (
+            {'hooks': {'cronitor': 'https://example.com'}},
+            {'hooks': {'cronitor': {'ping_url': 'https://example.com'}}},
+        ),
+        (
+            {'hooks': {'pagerduty': 'https://example.com'}},
+            {'hooks': {'pagerduty': {'integration_key': 'https://example.com'}}},
+        ),
+        (
+            {'hooks': {'cronhub': 'https://example.com'}},
+            {'hooks': {'cronhub': {'ping_url': 'https://example.com'}}},
+        ),
     ),
 )
 def test_normalize_applies_hard_coded_normalization_to_config(config, expected_config):

tests/unit/hooks/test_cronhub.py

@@ -4,45 +4,57 @@ from borgmatic.hooks import cronhub as module
 
 
 def test_ping_monitor_rewrites_ping_url_for_start_state():
-    ping_url = 'https://example.com/start/abcdef'
+    hook_config = {'ping_url': 'https://example.com/start/abcdef'}
     flexmock(module.requests).should_receive('get').with_args('https://example.com/start/abcdef')
 
     module.ping_monitor(
-        ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False
+        hook_config,
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
     )
 
 
 def test_ping_monitor_rewrites_ping_url_and_state_for_start_state():
-    ping_url = 'https://example.com/ping/abcdef'
+    hook_config = {'ping_url': 'https://example.com/ping/abcdef'}
     flexmock(module.requests).should_receive('get').with_args('https://example.com/start/abcdef')
 
     module.ping_monitor(
-        ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False
+        hook_config,
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
     )
 
 
 def test_ping_monitor_rewrites_ping_url_for_finish_state():
-    ping_url = 'https://example.com/start/abcdef'
+    hook_config = {'ping_url': 'https://example.com/start/abcdef'}
     flexmock(module.requests).should_receive('get').with_args('https://example.com/finish/abcdef')
 
     module.ping_monitor(
-        ping_url, 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False
+        hook_config,
+        'config.yaml',
+        module.monitor.State.FINISH,
+        monitoring_log_level=1,
+        dry_run=False,
     )
 
 
 def test_ping_monitor_rewrites_ping_url_for_fail_state():
-    ping_url = 'https://example.com/start/abcdef'
+    hook_config = {'ping_url': 'https://example.com/start/abcdef'}
     flexmock(module.requests).should_receive('get').with_args('https://example.com/fail/abcdef')
 
     module.ping_monitor(
-        ping_url, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
     )
 
 
 def test_ping_monitor_dry_run_does_not_hit_ping_url():
-    ping_url = 'https://example.com'
+    hook_config = {'ping_url': 'https://example.com'}
     flexmock(module.requests).should_receive('get').never()
 
     module.ping_monitor(
-        ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True
+        hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True
     )

tests/unit/hooks/test_cronitor.py

@@ -4,36 +4,44 @@ from borgmatic.hooks import cronitor as module
 
 
 def test_ping_monitor_hits_ping_url_for_start_state():
-    ping_url = 'https://example.com'
+    hook_config = {'ping_url': 'https://example.com'}
-    flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, 'run'))
+    flexmock(module.requests).should_receive('get').with_args('https://example.com/run')
 
     module.ping_monitor(
-        ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False
+        hook_config,
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
     )
 
 
 def test_ping_monitor_hits_ping_url_for_finish_state():
-    ping_url = 'https://example.com'
+    hook_config = {'ping_url': 'https://example.com'}
-    flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, 'complete'))
+    flexmock(module.requests).should_receive('get').with_args('https://example.com/complete')
 
     module.ping_monitor(
-        ping_url, 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False
+        hook_config,
+        'config.yaml',
+        module.monitor.State.FINISH,
+        monitoring_log_level=1,
+        dry_run=False,
     )
 
 
 def test_ping_monitor_hits_ping_url_for_fail_state():
-    ping_url = 'https://example.com'
+    hook_config = {'ping_url': 'https://example.com'}
-    flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, 'fail'))
+    flexmock(module.requests).should_receive('get').with_args('https://example.com/fail')
 
     module.ping_monitor(
-        ping_url, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
     )
 
 
 def test_ping_monitor_dry_run_does_not_hit_ping_url():
-    ping_url = 'https://example.com'
+    hook_config = {'ping_url': 'https://example.com'}
     flexmock(module.requests).should_receive('get').never()
 
     module.ping_monitor(
-        ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True
+        hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True
     )

tests/unit/hooks/test_healthchecks.py

@@ -62,13 +62,13 @@ def test_format_buffered_logs_for_payload_without_handler_produces_empty_payload
 
 def test_ping_monitor_hits_ping_url_for_start_state():
     flexmock(module).should_receive('Forgetful_buffering_handler')
-    ping_url = 'https://example.com'
+    hook_config = {'ping_url': 'https://example.com'}
     flexmock(module.requests).should_receive('post').with_args(
-        '{}/{}'.format(ping_url, 'start'), data=''.encode('utf-8')
+        'https://example.com/start', data=''.encode('utf-8')
     )
 
     module.ping_monitor(
-        ping_url,
+        hook_config,
         'config.yaml',
         state=module.monitor.State.START,
         monitoring_log_level=1,
@@ -77,15 +77,15 @@ def test_ping_monitor_hits_ping_url_for_start_state():
 
 
 def test_ping_monitor_hits_ping_url_for_finish_state():
-    ping_url = 'https://example.com'
+    hook_config = {'ping_url': 'https://example.com'}
     payload = 'data'
     flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
     flexmock(module.requests).should_receive('post').with_args(
-        ping_url, data=payload.encode('utf-8')
+        'https://example.com', data=payload.encode('utf-8')
     )
 
     module.ping_monitor(
-        ping_url,
+        hook_config,
         'config.yaml',
         state=module.monitor.State.FINISH,
         monitoring_log_level=1,
@@ -94,15 +94,15 @@ def test_ping_monitor_hits_ping_url_for_finish_state():
 
 
 def test_ping_monitor_hits_ping_url_for_fail_state():
-    ping_url = 'https://example.com'
+    hook_config = {'ping_url': 'https://example.com'}
     payload = 'data'
     flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
     flexmock(module.requests).should_receive('post').with_args(
-        '{}/{}'.format(ping_url, 'fail'), data=payload.encode('utf')
+        'https://example.com/fail', data=payload.encode('utf')
     )
 
     module.ping_monitor(
-        ping_url,
+        hook_config,
         'config.yaml',
         state=module.monitor.State.FAIL,
         monitoring_log_level=1,
@@ -111,15 +111,15 @@ def test_ping_monitor_hits_ping_url_for_fail_state():
 
 
 def test_ping_monitor_with_ping_uuid_hits_corresponding_url():
-    ping_uuid = 'abcd-efgh-ijkl-mnop'
+    hook_config = {'ping_url': 'abcd-efgh-ijkl-mnop'}
     payload = 'data'
     flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
     flexmock(module.requests).should_receive('post').with_args(
-        'https://hc-ping.com/{}'.format(ping_uuid), data=payload.encode('utf-8')
+        'https://hc-ping.com/{}'.format(hook_config['ping_url']), data=payload.encode('utf-8')
     )
 
     module.ping_monitor(
-        ping_uuid,
+        hook_config,
         'config.yaml',
         state=module.monitor.State.FINISH,
         monitoring_log_level=1,
@@ -129,11 +129,11 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url():
 
 def test_ping_monitor_dry_run_does_not_hit_ping_url():
     flexmock(module).should_receive('Forgetful_buffering_handler')
-    ping_url = 'https://example.com'
+    hook_config = {'ping_url': 'https://example.com'}
     flexmock(module.requests).should_receive('post').never()
 
     module.ping_monitor(
-        ping_url,
+        hook_config,
         'config.yaml',
         state=module.monitor.State.START,
         monitoring_log_level=1,

tests/unit/hooks/test_pagerduty.py

@@ -7,7 +7,11 @@ def test_ping_monitor_ignores_start_state():
     flexmock(module.requests).should_receive('post').never()
 
     module.ping_monitor(
-        'abc123', 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False
+        {'integration_key': 'abc123'},
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
     )
 
 
@@ -15,7 +19,11 @@ def test_ping_monitor_ignores_finish_state():
     flexmock(module.requests).should_receive('post').never()
 
     module.ping_monitor(
-        'abc123', 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False
+        {'integration_key': 'abc123'},
+        'config.yaml',
+        module.monitor.State.FINISH,
+        monitoring_log_level=1,
+        dry_run=False,
     )
 
 
@@ -23,7 +31,11 @@ def test_ping_monitor_calls_api_for_fail_state():
     flexmock(module.requests).should_receive('post')
 
     module.ping_monitor(
-        'abc123', 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+        {'integration_key': 'abc123'},
+        'config.yaml',
+        module.monitor.State.FAIL,
+        monitoring_log_level=1,
+        dry_run=False,
     )
 
 
@@ -31,5 +43,9 @@ def test_ping_monitor_dry_run_does_not_call_api():
     flexmock(module.requests).should_receive('post').never()
 
     module.ping_monitor(
-        'abc123', 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=True
+        {'integration_key': 'abc123'},
+        'config.yaml',
+        module.monitor.State.FAIL,
+        monitoring_log_level=1,
+        dry_run=True,
     )