From e87ebab62581955888051b8e29f569198ea4afea Mon Sep 17 00:00:00 2001 From: jetchirag Date: Sat, 25 Mar 2023 19:04:46 +0530 Subject: [PATCH] Initial schema modification and ping_monitor params changes Signed-off-by: jetchirag --- borgmatic/commands/borgmatic.py | 5 ++++ borgmatic/config/normalize.py | 7 +++++ borgmatic/config/schema.yaml | 34 +++++++++++++++++------ borgmatic/hooks/cronhub.py | 2 +- borgmatic/hooks/cronitor.py | 8 ++++-- borgmatic/hooks/healthchecks.py | 2 +- borgmatic/hooks/ntfy.py | 2 +- borgmatic/hooks/pagerduty.py | 2 +- tests/unit/hooks/test_cronhub.py | 26 +++++++++++++++--- tests/unit/hooks/test_cronitor.py | 39 ++++++++++++++++++++------- tests/unit/hooks/test_healthchecks.py | 12 +++++++++ tests/unit/hooks/test_ntfy.py | 12 +++++++++ tests/unit/hooks/test_pagerduty.py | 6 +++++ 13 files changed, 130 insertions(+), 27 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fbea260d..7b2f1ef5 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -66,6 +66,7 @@ def run_configuration(config_filename, config, arguments): error_repository = '' using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) + action_name = list(arguments.keys())[0] try: local_borg_version = borg_version.local_borg_version(storage, local_path) @@ -94,6 +95,7 @@ def run_configuration(config_filename, config, arguments): monitor.State.START, monitoring_log_level, global_arguments.dry_run, + action_name, ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): @@ -163,6 +165,7 @@ def run_configuration(config_filename, config, arguments): monitor.State.LOG, monitoring_log_level, global_arguments.dry_run, + action_name, ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): @@ -182,6 +185,7 @@ def run_configuration(config_filename, config, arguments): monitor.State.FINISH, monitoring_log_level, global_arguments.dry_run, + action_name, ) dispatch.call_hooks( 'destroy_monitor', @@ -218,6 +222,7 @@ def run_configuration(config_filename, config, arguments): monitor.State.FAIL, monitoring_log_level, global_arguments.dry_run, + action_name, ) dispatch.call_hooks( 'destroy_monitor', diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index a143a192..9079500e 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -27,6 +27,13 @@ def normalize(config_filename, config): cronitor = hooks.get('cronitor') if isinstance(cronitor, str): config['hooks']['cronitor'] = {'ping_url': cronitor} + if isinstance(cronitor, dict) and 'ping_url' in cronitor: + config['hooks']['cronitor'] = { + 'create': cronitor['ping_url'], + 'prune': cronitor['ping_url'], + 'compact': cronitor['ping_url'], + 'check': cronitor['ping_url'], + } pagerduty = hooks.get('pagerduty') if isinstance(pagerduty, str): diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d4d57ab6..b1a5ddbe 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1227,20 +1227,38 @@ properties: service. See borgmatic monitoring documentation for details. cronitor: type: object - required: ['ping_url'] additionalProperties: false properties: + create: + type: string + description: | + Cronitor ping URL to notify when a backup + begins, ends, or errors. + example: https://cronitor.link/d3x0c1 + prune: + type: string + description: | + Cronitor ping URL to notify when a prune action + begins, ends, or errors. + example: https://cronitor.link/d3x0c1 + compact: + type: string + description: | + Cronitor ping URL to notify when a compact action + begins, ends, or errors. + example: https://cronitor.link/d3x0c1 + check: + type: string + description: | + Cronitor ping URL to notify when a check action + begins, ends, or errors. + example: https://cronitor.link/d3x0c1 ping_url: type: string description: | - Cronitor ping URL to notify when a backup begins, - ends, or errors. + If this is set, other properties will be ignored + and replaced by this value. example: https://cronitor.link/d3x0c1 - description: | - Configuration for a monitoring integration with Cronitor. - Create an account at https://cronitor.io if you'd - like to use this service. See borgmatic monitoring - documentation for details. pagerduty: type: object required: ['integration_key'] diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index cd0ffa5c..af9cabe3 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -22,7 +22,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index 633b4c3c..563c376a 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -22,7 +22,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. @@ -34,7 +34,11 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' - ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state]) + try: + ping_url = '{}/{}'.format(hook_config[action_name], MONITOR_STATE_TO_CRONITOR[state]) + except KeyError: + print('KeyError') + return logger.info( '{}: Pinging Cronitor {}{}'.format(config_filename, state.name.lower(), dry_run_label) diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index 6ad8449f..4ca6a89f 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -90,7 +90,7 @@ def initialize_monitor(hook_config, config_filename, monitoring_log_level, dry_r ) -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' Ping the configured Healthchecks URL or UUID, modified with the monitor.State. Use the given configuration filename in any log entries, and log to Healthchecks with the giving log level. diff --git a/borgmatic/hooks/ntfy.py b/borgmatic/hooks/ntfy.py index 8a6f0fb8..b460bfd4 100644 --- a/borgmatic/hooks/ntfy.py +++ b/borgmatic/hooks/ntfy.py @@ -14,7 +14,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' Ping the configured Ntfy topic. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. diff --git a/borgmatic/hooks/pagerduty.py b/borgmatic/hooks/pagerduty.py index fbb67fbf..de25e681 100644 --- a/borgmatic/hooks/pagerduty.py +++ b/borgmatic/hooks/pagerduty.py @@ -21,7 +21,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' If this is an error state, create a PagerDuty event with the configured integration key. Use the given configuration filename in any log entries. If this is a dry run, then don't actually diff --git a/tests/unit/hooks/test_cronhub.py b/tests/unit/hooks/test_cronhub.py index f470b88e..be0f032d 100644 --- a/tests/unit/hooks/test_cronhub.py +++ b/tests/unit/hooks/test_cronhub.py @@ -15,6 +15,7 @@ def test_ping_monitor_rewrites_ping_url_for_start_state(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -30,6 +31,7 @@ def test_ping_monitor_rewrites_ping_url_and_state_for_start_state(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -45,6 +47,7 @@ def test_ping_monitor_rewrites_ping_url_for_finish_state(): module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -55,7 +58,12 @@ def test_ping_monitor_rewrites_ping_url_for_fail_state(): ).and_return(flexmock(ok=True)) module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + action_name='create', ) @@ -64,7 +72,12 @@ def test_ping_monitor_dry_run_does_not_hit_ping_url(): flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True + hook_config, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=True, + action_name='create', ) @@ -81,6 +94,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -101,6 +115,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -108,5 +123,10 @@ def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, + action_name='create', ) diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py index 7ec1e2e6..ba2fefbc 100644 --- a/tests/unit/hooks/test_cronitor.py +++ b/tests/unit/hooks/test_cronitor.py @@ -4,7 +4,7 @@ from borgmatic.hooks import cronitor as module def test_ping_monitor_hits_ping_url_for_start_state(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').with_args('https://example.com/run').and_return( flexmock(ok=True) ) @@ -15,11 +15,12 @@ def test_ping_monitor_hits_ping_url_for_start_state(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) def test_ping_monitor_hits_ping_url_for_finish_state(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').with_args( 'https://example.com/complete' ).and_return(flexmock(ok=True)) @@ -30,31 +31,42 @@ def test_ping_monitor_hits_ping_url_for_finish_state(): module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) def test_ping_monitor_hits_ping_url_for_fail_state(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').with_args( 'https://example.com/fail' ).and_return(flexmock(ok=True)) module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + action_name='create', ) def test_ping_monitor_dry_run_does_not_hit_ping_url(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True + hook_config, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=True, + action_name='create', ) def test_ping_monitor_with_connection_error_logs_warning(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').and_raise( module.requests.exceptions.ConnectionError ) @@ -66,11 +78,12 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) def test_ping_monitor_with_other_error_logs_warning(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} response = flexmock(ok=False) response.should_receive('raise_for_status').and_raise( module.requests.exceptions.RequestException @@ -86,12 +99,18 @@ def test_ping_monitor_with_other_error_logs_warning(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) def test_ping_monitor_with_unsupported_monitoring_state(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, + action_name='create', ) diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index d5779534..5af1ec8d 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -147,6 +147,7 @@ def test_ping_monitor_hits_ping_url_for_start_state(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -164,6 +165,7 @@ def test_ping_monitor_hits_ping_url_for_finish_state(): state=module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -181,6 +183,7 @@ def test_ping_monitor_hits_ping_url_for_fail_state(): state=module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -198,6 +201,7 @@ def test_ping_monitor_hits_ping_url_for_log_state(): state=module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -217,6 +221,7 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): state=module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -234,6 +239,7 @@ def test_ping_monitor_skips_ssl_verification_when_verify_tls_false(): state=module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -251,6 +257,7 @@ def test_ping_monitor_executes_ssl_verification_when_verify_tls_true(): state=module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -265,6 +272,7 @@ def test_ping_monitor_dry_run_does_not_hit_ping_url(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=True, + action_name='create', ) @@ -279,6 +287,7 @@ def test_ping_monitor_does_not_hit_ping_url_when_states_not_matching(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=True, + action_name='create', ) @@ -295,6 +304,7 @@ def test_ping_monitor_hits_ping_url_when_states_matching(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -312,6 +322,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -333,4 +344,5 @@ def test_ping_monitor_with_other_error_logs_warning(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) diff --git a/tests/unit/hooks/test_ntfy.py b/tests/unit/hooks/test_ntfy.py index 9731df7a..c1f75c0e 100644 --- a/tests/unit/hooks/test_ntfy.py +++ b/tests/unit/hooks/test_ntfy.py @@ -48,6 +48,7 @@ def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -69,6 +70,7 @@ def test_ping_monitor_with_auth_hits_hosted_ntfy_on_fail(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -87,6 +89,7 @@ def test_ping_monitor_auth_with_no_username_warning(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -105,6 +108,7 @@ def test_ping_monitor_auth_with_no_password_warning(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -118,6 +122,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): borgmatic.hooks.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -131,6 +136,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): borgmatic.hooks.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -148,6 +154,7 @@ def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -161,6 +168,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=True, + action_name='create', ) @@ -176,6 +184,7 @@ def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -193,6 +202,7 @@ def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): borgmatic.hooks.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -211,6 +221,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -233,4 +244,5 @@ def test_ping_monitor_with_other_error_logs_warning(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) diff --git a/tests/unit/hooks/test_pagerduty.py b/tests/unit/hooks/test_pagerduty.py index 0fccae00..659412dd 100644 --- a/tests/unit/hooks/test_pagerduty.py +++ b/tests/unit/hooks/test_pagerduty.py @@ -12,6 +12,7 @@ def test_ping_monitor_ignores_start_state(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -24,6 +25,7 @@ def test_ping_monitor_ignores_finish_state(): module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -36,6 +38,7 @@ def test_ping_monitor_calls_api_for_fail_state(): module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -48,6 +51,7 @@ def test_ping_monitor_dry_run_does_not_call_api(): module.monitor.State.FAIL, monitoring_log_level=1, dry_run=True, + action_name='create', ) @@ -63,6 +67,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -80,4 +85,5 @@ def test_ping_monitor_with_other_error_logs_warning(): module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', )