From 8fd46b8c70fde8d3a2ea66e8bf707474daf0ff54 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 1 Nov 2019 11:33:15 -0700 Subject: [PATCH] Monitor backups with Cronitor hook integration. --- NEWS | 4 ++++ borgmatic/commands/arguments.py | 4 ++-- borgmatic/commands/borgmatic.py | 11 +++++++++- borgmatic/config/schema.yaml | 7 +++++++ borgmatic/hooks/cronitor.py | 24 ++++++++++++++++++++++ borgmatic/hooks/healthchecks.py | 8 ++++---- docs/how-to/monitor-your-backups.md | 32 +++++++++++++++++++++++++---- setup.py | 2 +- tests/unit/hooks/test_cronitor.py | 17 +++++++++++++++ 9 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 borgmatic/hooks/cronitor.py create mode 100644 tests/unit/hooks/test_cronitor.py diff --git a/NEWS b/NEWS index bc5dbbf..6516b09 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.4.3 + * Monitor backups with Cronitor hook integration. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook + 1.4.2 * Extract files to a particular directory via "borgmatic extract --destination" flag. * Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index c5f900f..77ab7b2 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -273,7 +273,7 @@ def parse_arguments(*unparsed_arguments): '--repository', help='Path of repository to extract, defaults to the configured repository if there is only one', ) - extract_group.add_argument('--archive', help='Name of archive to extract, required=True) + extract_group.add_argument('--archive', help='Name of archive to extract', required=True) extract_group.add_argument( '--path', '--restore-path', @@ -311,7 +311,7 @@ def parse_arguments(*unparsed_arguments): '--repository', help='Path of repository to restore from, defaults to the configured repository if there is only one', ) - restore_group.add_argument('--archive', help='Name of archive to restore from, required=True) + restore_group.add_argument('--archive', help='Name of archive to restore from', required=True) restore_group.add_argument( '--database', metavar='NAME', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fce102c..0941370 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list from borgmatic.borg import prune as borg_prune from borgmatic.commands.arguments import parse_arguments from borgmatic.config import checks, collect, convert, validate -from borgmatic.hooks import command, healthchecks, postgresql +from borgmatic.hooks import command, cronitor, healthchecks, postgresql from borgmatic.logger import configure_logging, should_do_markup from borgmatic.signals import configure_signals from borgmatic.verbosity import verbosity_to_log_level @@ -56,6 +56,9 @@ def run_configuration(config_filename, config, arguments): healthchecks.ping_healthchecks( hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start' ) + cronitor.ping_cronitor( + hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run' + ) command.execute_hook( hooks.get('before_backup'), hooks.get('umask'), @@ -108,6 +111,9 @@ def run_configuration(config_filename, config, arguments): healthchecks.ping_healthchecks( hooks.get('healthchecks'), config_filename, global_arguments.dry_run ) + cronitor.ping_cronitor( + hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete' + ) except (OSError, CalledProcessError) as error: encountered_error = error yield from make_error_log_records( @@ -129,6 +135,9 @@ def run_configuration(config_filename, config, arguments): healthchecks.ping_healthchecks( hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail' ) + cronitor.ping_cronitor( + hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail' + ) except (OSError, CalledProcessError) as error: yield from make_error_log_records( '{}: Error running on-error hook'.format(config_filename), error diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index e113580..1af9334 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -430,6 +430,13 @@ map: Create an account at https://healthchecks.io if you'd like to use this service. example: https://hc-ping.com/your-uuid-here + cronitor: + type: str + desc: | + Cronitor ping URL to notify when a backup begins, ends, or errors. Create an + account at https://cronitor.io if you'd like to use this service. + example: + https://cronitor.link/d3x0c1 before_everything: seq: - type: str diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py new file mode 100644 index 0000000..f7042dc --- /dev/null +++ b/borgmatic/hooks/cronitor.py @@ -0,0 +1,24 @@ +import logging + +import requests + +logger = logging.getLogger(__name__) + + +def ping_cronitor(ping_url, config_filename, dry_run, append): + ''' + Ping the given Cronitor URL, appending the append string. Use the given configuration filename + in any log entries. If this is a dry run, then don't actually ping anything. + ''' + if not ping_url: + logger.debug('{}: No Cronitor hook set'.format(config_filename)) + return + + dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' + ping_url = '{}/{}'.format(ping_url, append) + + logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label)) + logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url)) + + logging.getLogger('urllib3').setLevel(logging.ERROR) + requests.get(ping_url) diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index dc9a196..645f512 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -7,12 +7,12 @@ logger = logging.getLogger(__name__) def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None): ''' - Ping the given healthchecks.io URL or UUID, appending the append string if any. Use the given + Ping the given Healthchecks URL or UUID, appending the append string if any. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. ''' if not ping_url_or_uuid: - logger.debug('{}: No healthchecks hook set'.format(config_filename)) + logger.debug('{}: No Healthchecks hook set'.format(config_filename)) return ping_url = ( @@ -26,11 +26,11 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None): ping_url = '{}/{}'.format(ping_url, append) logger.info( - '{}: Pinging healthchecks.io{}{}'.format( + '{}: Pinging Healthchecks{}{}'.format( config_filename, ' ' + append if append else '', dry_run_label ) ) - logger.debug('{}: Using healthchecks.io ping URL {}'.format(config_filename, ping_url)) + logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url)) logging.getLogger('urllib3').setLevel(logging.ERROR) requests.get(ping_url) diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 99cb563..894d91b 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -26,12 +26,15 @@ alert. But note that if borgmatic doesn't actually run, this alert won't fire. See [error hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks) below for how to configure this. -4. **borgmatic Healthchecks hook**: This feature integrates with the -[Healthchecks](https://healthchecks.io/) service, and pings Healthchecks -whenever borgmatic runs. That way, Healthchecks can alert you when something -goes wrong or it doesn't hear from borgmatic for a configured interval. See +4. **borgmatic monitoring hooks**: This feature integrates with monitoring +services like [Healthchecks](https://healthchecks.io/) and +[Cronitor](https://cronitor.io), and pings these services whenever borgmatic +runs. That way, you'll receive an alert when something goes wrong or the +service doesn't hear from borgmatic for a configured interval. See [Healthchecks hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook) +and [Cronitor +hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook) below for how to configure this. 3. **Third-party monitoring software**: You can use traditional monitoring software to consume borgmatic JSON output and track when the last @@ -115,6 +118,27 @@ mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail or it doesn't hear from borgmatic for a certain period of time. +## Cronitor hook + +[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks +for websites, services and APIs", and borgmatic has built-in +integration with it. Once you create a Cronitor account and cron job monitor on +their site, all you need to do is configure borgmatic with the unique "Ping +API URL" for your monitor. Here's an example: + + +```yaml +hooks: + cronitor: https://cronitor.link/d3x0c1 +``` + +With this hook in place, borgmatic will ping your Cronitor monitor when a +backup begins, ends, or errors. Then you can configure Cronitor to notify you +by a [variety of +mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups +fail or it doesn't hear from borgmatic for a certain period of time. + + ## Scripting borgmatic To consume the output of borgmatic in other software, you can include an diff --git a/setup.py b/setup.py index 8610ef4..849de4b 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.4.2' +VERSION = '1.4.3' setup( diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py new file mode 100644 index 0000000..aad8d66 --- /dev/null +++ b/tests/unit/hooks/test_cronitor.py @@ -0,0 +1,17 @@ +from flexmock import flexmock + +from borgmatic.hooks import cronitor as module + + +def test_ping_cronitor_hits_ping_url(): + ping_url = 'https://example.com' + append = 'failed-so-hard' + flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append)) + + module.ping_cronitor(ping_url, 'config.yaml', dry_run=False, append=append) + + +def test_ping_cronitor_without_ping_url_does_not_raise(): + flexmock(module.requests).should_receive('get').never() + + module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')