From e87ebab62581955888051b8e29f569198ea4afea Mon Sep 17 00:00:00 2001 From: jetchirag Date: Sat, 25 Mar 2023 19:04:46 +0530 Subject: [PATCH 1/6] Initial schema modification and ping_monitor params changes Signed-off-by: jetchirag --- borgmatic/commands/borgmatic.py | 5 ++++ borgmatic/config/normalize.py | 7 +++++ borgmatic/config/schema.yaml | 34 +++++++++++++++++------ borgmatic/hooks/cronhub.py | 2 +- borgmatic/hooks/cronitor.py | 8 ++++-- borgmatic/hooks/healthchecks.py | 2 +- borgmatic/hooks/ntfy.py | 2 +- borgmatic/hooks/pagerduty.py | 2 +- tests/unit/hooks/test_cronhub.py | 26 +++++++++++++++--- tests/unit/hooks/test_cronitor.py | 39 ++++++++++++++++++++------- tests/unit/hooks/test_healthchecks.py | 12 +++++++++ tests/unit/hooks/test_ntfy.py | 12 +++++++++ tests/unit/hooks/test_pagerduty.py | 6 +++++ 13 files changed, 130 insertions(+), 27 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fbea260d..7b2f1ef5 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -66,6 +66,7 @@ def run_configuration(config_filename, config, arguments): error_repository = '' using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) + action_name = list(arguments.keys())[0] try: local_borg_version = borg_version.local_borg_version(storage, local_path) @@ -94,6 +95,7 @@ def run_configuration(config_filename, config, arguments): monitor.State.START, monitoring_log_level, global_arguments.dry_run, + action_name, ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): @@ -163,6 +165,7 @@ def run_configuration(config_filename, config, arguments): monitor.State.LOG, monitoring_log_level, global_arguments.dry_run, + action_name, ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): @@ -182,6 +185,7 @@ def run_configuration(config_filename, config, arguments): monitor.State.FINISH, monitoring_log_level, global_arguments.dry_run, + action_name, ) dispatch.call_hooks( 'destroy_monitor', @@ -218,6 +222,7 @@ def run_configuration(config_filename, config, arguments): monitor.State.FAIL, monitoring_log_level, global_arguments.dry_run, + action_name, ) dispatch.call_hooks( 'destroy_monitor', diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index a143a192..9079500e 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -27,6 +27,13 @@ def normalize(config_filename, config): cronitor = hooks.get('cronitor') if isinstance(cronitor, str): config['hooks']['cronitor'] = {'ping_url': cronitor} + if isinstance(cronitor, dict) and 'ping_url' in cronitor: + config['hooks']['cronitor'] = { + 'create': cronitor['ping_url'], + 'prune': cronitor['ping_url'], + 'compact': cronitor['ping_url'], + 'check': cronitor['ping_url'], + } pagerduty = hooks.get('pagerduty') if isinstance(pagerduty, str): diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d4d57ab6..b1a5ddbe 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1227,20 +1227,38 @@ properties: service. See borgmatic monitoring documentation for details. cronitor: type: object - required: ['ping_url'] additionalProperties: false properties: + create: + type: string + description: | + Cronitor ping URL to notify when a backup + begins, ends, or errors. + example: https://cronitor.link/d3x0c1 + prune: + type: string + description: | + Cronitor ping URL to notify when a prune action + begins, ends, or errors. + example: https://cronitor.link/d3x0c1 + compact: + type: string + description: | + Cronitor ping URL to notify when a compact action + begins, ends, or errors. + example: https://cronitor.link/d3x0c1 + check: + type: string + description: | + Cronitor ping URL to notify when a check action + begins, ends, or errors. + example: https://cronitor.link/d3x0c1 ping_url: type: string description: | - Cronitor ping URL to notify when a backup begins, - ends, or errors. + If this is set, other properties will be ignored + and replaced by this value. example: https://cronitor.link/d3x0c1 - description: | - Configuration for a monitoring integration with Cronitor. - Create an account at https://cronitor.io if you'd - like to use this service. See borgmatic monitoring - documentation for details. pagerduty: type: object required: ['integration_key'] diff --git a/borgmatic/hooks/cronhub.py b/borgmatic/hooks/cronhub.py index cd0ffa5c..af9cabe3 100644 --- a/borgmatic/hooks/cronhub.py +++ b/borgmatic/hooks/cronhub.py @@ -22,7 +22,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index 633b4c3c..563c376a 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -22,7 +22,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. @@ -34,7 +34,11 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' - ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state]) + try: + ping_url = '{}/{}'.format(hook_config[action_name], MONITOR_STATE_TO_CRONITOR[state]) + except KeyError: + print('KeyError') + return logger.info( '{}: Pinging Cronitor {}{}'.format(config_filename, state.name.lower(), dry_run_label) diff --git a/borgmatic/hooks/healthchecks.py b/borgmatic/hooks/healthchecks.py index 6ad8449f..4ca6a89f 100644 --- a/borgmatic/hooks/healthchecks.py +++ b/borgmatic/hooks/healthchecks.py @@ -90,7 +90,7 @@ def initialize_monitor(hook_config, config_filename, monitoring_log_level, dry_r ) -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' Ping the configured Healthchecks URL or UUID, modified with the monitor.State. Use the given configuration filename in any log entries, and log to Healthchecks with the giving log level. diff --git a/borgmatic/hooks/ntfy.py b/borgmatic/hooks/ntfy.py index 8a6f0fb8..b460bfd4 100644 --- a/borgmatic/hooks/ntfy.py +++ b/borgmatic/hooks/ntfy.py @@ -14,7 +14,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' Ping the configured Ntfy topic. Use the given configuration filename in any log entries. If this is a dry run, then don't actually ping anything. diff --git a/borgmatic/hooks/pagerduty.py b/borgmatic/hooks/pagerduty.py index fbb67fbf..de25e681 100644 --- a/borgmatic/hooks/pagerduty.py +++ b/borgmatic/hooks/pagerduty.py @@ -21,7 +21,7 @@ def initialize_monitor( pass -def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run): +def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run, action_name): ''' If this is an error state, create a PagerDuty event with the configured integration key. Use the given configuration filename in any log entries. If this is a dry run, then don't actually diff --git a/tests/unit/hooks/test_cronhub.py b/tests/unit/hooks/test_cronhub.py index f470b88e..be0f032d 100644 --- a/tests/unit/hooks/test_cronhub.py +++ b/tests/unit/hooks/test_cronhub.py @@ -15,6 +15,7 @@ def test_ping_monitor_rewrites_ping_url_for_start_state(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -30,6 +31,7 @@ def test_ping_monitor_rewrites_ping_url_and_state_for_start_state(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -45,6 +47,7 @@ def test_ping_monitor_rewrites_ping_url_for_finish_state(): module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -55,7 +58,12 @@ def test_ping_monitor_rewrites_ping_url_for_fail_state(): ).and_return(flexmock(ok=True)) module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + action_name='create', ) @@ -64,7 +72,12 @@ def test_ping_monitor_dry_run_does_not_hit_ping_url(): flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True + hook_config, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=True, + action_name='create', ) @@ -81,6 +94,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -101,6 +115,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -108,5 +123,10 @@ def test_ping_monitor_with_unsupported_monitoring_state(): hook_config = {'ping_url': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, + action_name='create', ) diff --git a/tests/unit/hooks/test_cronitor.py b/tests/unit/hooks/test_cronitor.py index 7ec1e2e6..ba2fefbc 100644 --- a/tests/unit/hooks/test_cronitor.py +++ b/tests/unit/hooks/test_cronitor.py @@ -4,7 +4,7 @@ from borgmatic.hooks import cronitor as module def test_ping_monitor_hits_ping_url_for_start_state(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').with_args('https://example.com/run').and_return( flexmock(ok=True) ) @@ -15,11 +15,12 @@ def test_ping_monitor_hits_ping_url_for_start_state(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) def test_ping_monitor_hits_ping_url_for_finish_state(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').with_args( 'https://example.com/complete' ).and_return(flexmock(ok=True)) @@ -30,31 +31,42 @@ def test_ping_monitor_hits_ping_url_for_finish_state(): module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) def test_ping_monitor_hits_ping_url_for_fail_state(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').with_args( 'https://example.com/fail' ).and_return(flexmock(ok=True)) module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + hook_config, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + action_name='create', ) def test_ping_monitor_dry_run_does_not_hit_ping_url(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True + hook_config, + 'config.yaml', + module.monitor.State.START, + monitoring_log_level=1, + dry_run=True, + action_name='create', ) def test_ping_monitor_with_connection_error_logs_warning(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').and_raise( module.requests.exceptions.ConnectionError ) @@ -66,11 +78,12 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) def test_ping_monitor_with_other_error_logs_warning(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} response = flexmock(ok=False) response.should_receive('raise_for_status').and_raise( module.requests.exceptions.RequestException @@ -86,12 +99,18 @@ def test_ping_monitor_with_other_error_logs_warning(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) def test_ping_monitor_with_unsupported_monitoring_state(): - hook_config = {'ping_url': 'https://example.com'} + hook_config = {'create': 'https://example.com'} flexmock(module.requests).should_receive('get').never() module.ping_monitor( - hook_config, 'config.yaml', module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + hook_config, + 'config.yaml', + module.monitor.State.LOG, + monitoring_log_level=1, + dry_run=False, + action_name='create', ) diff --git a/tests/unit/hooks/test_healthchecks.py b/tests/unit/hooks/test_healthchecks.py index d5779534..5af1ec8d 100644 --- a/tests/unit/hooks/test_healthchecks.py +++ b/tests/unit/hooks/test_healthchecks.py @@ -147,6 +147,7 @@ def test_ping_monitor_hits_ping_url_for_start_state(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -164,6 +165,7 @@ def test_ping_monitor_hits_ping_url_for_finish_state(): state=module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -181,6 +183,7 @@ def test_ping_monitor_hits_ping_url_for_fail_state(): state=module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -198,6 +201,7 @@ def test_ping_monitor_hits_ping_url_for_log_state(): state=module.monitor.State.LOG, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -217,6 +221,7 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url(): state=module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -234,6 +239,7 @@ def test_ping_monitor_skips_ssl_verification_when_verify_tls_false(): state=module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -251,6 +257,7 @@ def test_ping_monitor_executes_ssl_verification_when_verify_tls_true(): state=module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -265,6 +272,7 @@ def test_ping_monitor_dry_run_does_not_hit_ping_url(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=True, + action_name='create', ) @@ -279,6 +287,7 @@ def test_ping_monitor_does_not_hit_ping_url_when_states_not_matching(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=True, + action_name='create', ) @@ -295,6 +304,7 @@ def test_ping_monitor_hits_ping_url_when_states_matching(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -312,6 +322,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -333,4 +344,5 @@ def test_ping_monitor_with_other_error_logs_warning(): state=module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) diff --git a/tests/unit/hooks/test_ntfy.py b/tests/unit/hooks/test_ntfy.py index 9731df7a..c1f75c0e 100644 --- a/tests/unit/hooks/test_ntfy.py +++ b/tests/unit/hooks/test_ntfy.py @@ -48,6 +48,7 @@ def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -69,6 +70,7 @@ def test_ping_monitor_with_auth_hits_hosted_ntfy_on_fail(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -87,6 +89,7 @@ def test_ping_monitor_auth_with_no_username_warning(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -105,6 +108,7 @@ def test_ping_monitor_auth_with_no_password_warning(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -118,6 +122,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): borgmatic.hooks.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -131,6 +136,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): borgmatic.hooks.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -148,6 +154,7 @@ def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -161,6 +168,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=True, + action_name='create', ) @@ -176,6 +184,7 @@ def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -193,6 +202,7 @@ def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): borgmatic.hooks.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -211,6 +221,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -233,4 +244,5 @@ def test_ping_monitor_with_other_error_logs_warning(): borgmatic.hooks.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) diff --git a/tests/unit/hooks/test_pagerduty.py b/tests/unit/hooks/test_pagerduty.py index 0fccae00..659412dd 100644 --- a/tests/unit/hooks/test_pagerduty.py +++ b/tests/unit/hooks/test_pagerduty.py @@ -12,6 +12,7 @@ def test_ping_monitor_ignores_start_state(): module.monitor.State.START, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -24,6 +25,7 @@ def test_ping_monitor_ignores_finish_state(): module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -36,6 +38,7 @@ def test_ping_monitor_calls_api_for_fail_state(): module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -48,6 +51,7 @@ def test_ping_monitor_dry_run_does_not_call_api(): module.monitor.State.FAIL, monitoring_log_level=1, dry_run=True, + action_name='create', ) @@ -63,6 +67,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) @@ -80,4 +85,5 @@ def test_ping_monitor_with_other_error_logs_warning(): module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False, + action_name='create', ) -- 2.40.1 From c3cc5cc95c3a7ad84084dd955e2e76bf30134894 Mon Sep 17 00:00:00 2001 From: jetchirag Date: Sat, 25 Mar 2023 20:04:19 +0530 Subject: [PATCH 2/6] Minor bug fixes, removed debug print and restored cronitor description in schema Signed-off-by: jetchirag --- borgmatic/commands/borgmatic.py | 2 +- borgmatic/config/schema.yaml | 5 +++++ borgmatic/hooks/cronitor.py | 1 - 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 7b2f1ef5..c20b3056 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -66,7 +66,7 @@ def run_configuration(config_filename, config, arguments): error_repository = '' using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) - action_name = list(arguments.keys())[0] + action_name = next(iter(arguments)) try: local_borg_version = borg_version.local_borg_version(storage, local_path) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index b1a5ddbe..f1ccf7fc 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1259,6 +1259,11 @@ properties: If this is set, other properties will be ignored and replaced by this value. example: https://cronitor.link/d3x0c1 + description: | + Configuration for a monitoring integration with Cronitor. + Create an account at https://cronitor.io if you'd + like to use this service. See borgmatic monitoring + documentation for details. pagerduty: type: object required: ['integration_key'] diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index 563c376a..0818a5a8 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -37,7 +37,6 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ try: ping_url = '{}/{}'.format(hook_config[action_name], MONITOR_STATE_TO_CRONITOR[state]) except KeyError: - print('KeyError') return logger.info( -- 2.40.1 From 6ce6367a26e6ac2545329de979b2fc648e5552f8 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 29 Mar 2023 03:32:05 +0530 Subject: [PATCH 3/6] --skip-monitoring flag Signed-off-by: Chirag Aggarwal --- borgmatic/commands/arguments.py | 7 +++++++ borgmatic/commands/borgmatic.py | 14 ++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 59751b08..11ef33b3 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -145,6 +145,13 @@ def make_parsers(): action='store_true', help='Go through the motions, but do not actually write to any repositories', ) + global_group.add_argument( + '-nm', + '--skip-monitoring', + dest='skip_monitoring', + action='store_true', + help='Skip reporting any data to the configured monitoring services', + ) global_group.add_argument( '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output' ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 8dd1c618..a6f56925 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -67,6 +67,12 @@ def run_configuration(config_filename, config, arguments): using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) action_name = next(iter(arguments)) + skip_monitoring = global_arguments.skip_monitoring + + # action_names = [action for action in arguments.keys() if action != 'global' and isinstance(action, str)] + + # print (action_names) + # return try: local_borg_version = borg_version.local_borg_version(storage, local_path) @@ -75,7 +81,7 @@ def run_configuration(config_filename, config, arguments): return try: - if using_primary_action: + if using_primary_action and not skip_monitoring: dispatch.call_hooks( 'initialize_monitor', hooks, @@ -84,7 +90,7 @@ def run_configuration(config_filename, config, arguments): monitoring_log_level, global_arguments.dry_run, ) - if using_primary_action: + if using_primary_action and not skip_monitoring: dispatch.call_hooks( 'ping_monitor', hooks, @@ -153,7 +159,7 @@ def run_configuration(config_filename, config, arguments): error_repository = repository_path try: - if using_primary_action: + if using_primary_action and not skip_monitoring: # send logs irrespective of error dispatch.call_hooks( 'ping_monitor', @@ -174,7 +180,7 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: try: - if using_primary_action: + if using_primary_action and not skip_monitoring: dispatch.call_hooks( 'ping_monitor', hooks, -- 2.40.1 From 2d728d57d7db6ab27c9b04c13b48abeceb6c4c41 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 29 Mar 2023 14:13:54 +0530 Subject: [PATCH 4/6] minor; log on skipping cronitor due to KeyError Signed-off-by: Chirag Aggarwal --- borgmatic/hooks/cronitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index 7533ab08..5fc7c5f3 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -35,8 +35,9 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' try: - ping_url = f"{hook_config[action_name]}/{MONITOR_STATE_TO_CRONITOR[state]}" + ping_url = f"{hook_config['sss']}/{MONITOR_STATE_TO_CRONITOR[state]}" except KeyError: + logger.debug(f'{config_filename}: Skipping Cronitor {state.name.lower()} ping due to unconfigured action: {action_name}') return logger.info(f'{config_filename}: Pinging Cronitor {state.name.lower()}{dry_run_label}') -- 2.40.1 From aa6f1e662349f502b6a3b90d832424334acba3bb Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 29 Mar 2023 15:55:17 +0530 Subject: [PATCH 5/6] Support multiple actions with ping_monitor Signed-off-by: Chirag Aggarwal --- borgmatic/commands/borgmatic.py | 143 +++++++++++++++++++------------- borgmatic/hooks/cronitor.py | 6 +- 2 files changed, 91 insertions(+), 58 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a6f56925..c2e7b656 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -66,13 +66,10 @@ def run_configuration(config_filename, config, arguments): error_repository = '' using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) - action_name = next(iter(arguments)) - skip_monitoring = global_arguments.skip_monitoring - - # action_names = [action for action in arguments.keys() if action != 'global' and isinstance(action, str)] - - # print (action_names) - # return + skip_monitoring = getattr(global_arguments, 'skip_monitoring', False) + action_names = [ + action for action in arguments.keys() if action != 'global' and isinstance(action, str) + ] try: local_borg_version = borg_version.local_borg_version(storage, local_path) @@ -90,17 +87,18 @@ def run_configuration(config_filename, config, arguments): monitoring_log_level, global_arguments.dry_run, ) - if using_primary_action and not skip_monitoring: - dispatch.call_hooks( - 'ping_monitor', - hooks, - config_filename, - monitor.MONITOR_HOOK_NAMES, - monitor.State.START, - monitoring_log_level, - global_arguments.dry_run, - action_name, - ) + + for action_name in action_names: + dispatch.call_hooks( + 'ping_monitor', + hooks, + config_filename, + monitor.MONITOR_HOOK_NAMES, + monitor.State.START, + monitoring_log_level, + global_arguments.dry_run, + action_name, + ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): return @@ -111,7 +109,9 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: repo_queue = Queue() for repo in location['repositories']: - repo_queue.put((repo, 0),) + repo_queue.put( + (repo, 0), + ) while not repo_queue.empty(): repository_path, retry_num = repo_queue.get() @@ -135,7 +135,9 @@ def run_configuration(config_filename, config, arguments): ) except (OSError, CalledProcessError, ValueError) as error: if retry_num < retries: - repo_queue.put((repository_path, retry_num + 1),) + repo_queue.put( + (repository_path, retry_num + 1), + ) tuple( # Consume the generator so as to trigger logging. log_error_records( f'{repository_path}: Error running actions for repository', @@ -161,16 +163,17 @@ def run_configuration(config_filename, config, arguments): try: if using_primary_action and not skip_monitoring: # send logs irrespective of error - dispatch.call_hooks( - 'ping_monitor', - hooks, - config_filename, - monitor.MONITOR_HOOK_NAMES, - monitor.State.LOG, - monitoring_log_level, - global_arguments.dry_run, - action_name, - ) + for action_name in action_names: + dispatch.call_hooks( + 'ping_monitor', + hooks, + config_filename, + monitor.MONITOR_HOOK_NAMES, + monitor.State.LOG, + monitoring_log_level, + global_arguments.dry_run, + action_name, + ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(config_filename, error): return @@ -181,16 +184,17 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: try: if using_primary_action and not skip_monitoring: - dispatch.call_hooks( - 'ping_monitor', - hooks, - config_filename, - monitor.MONITOR_HOOK_NAMES, - monitor.State.FINISH, - monitoring_log_level, - global_arguments.dry_run, - action_name, - ) + for action_name in action_names: + dispatch.call_hooks( + 'ping_monitor', + hooks, + config_filename, + monitor.MONITOR_HOOK_NAMES, + monitor.State.FINISH, + monitoring_log_level, + global_arguments.dry_run, + action_name, + ) dispatch.call_hooks( 'destroy_monitor', hooks, @@ -218,16 +222,17 @@ def run_configuration(config_filename, config, arguments): error=encountered_error, output=getattr(encountered_error, 'output', ''), ) - dispatch.call_hooks( - 'ping_monitor', - hooks, - config_filename, - monitor.MONITOR_HOOK_NAMES, - monitor.State.FAIL, - monitoring_log_level, - global_arguments.dry_run, - action_name, - ) + for action_name in action_names: + dispatch.call_hooks( + 'ping_monitor', + hooks, + config_filename, + monitor.MONITOR_HOOK_NAMES, + monitor.State.FAIL, + monitoring_log_level, + global_arguments.dry_run, + action_name, + ) dispatch.call_hooks( 'destroy_monitor', hooks, @@ -417,19 +422,39 @@ def run_actions( ) elif action_name == 'rlist': yield from borgmatic.actions.rlist.run_rlist( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'list': yield from borgmatic.actions.list.run_list( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'rinfo': yield from borgmatic.actions.rinfo.run_rinfo( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'info': yield from borgmatic.actions.info.run_info( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) elif action_name == 'break-lock': borgmatic.actions.break_lock.run_break_lock( @@ -442,7 +467,12 @@ def run_actions( ) elif action_name == 'borg': borgmatic.actions.borg.run_borg( - repository, storage, local_borg_version, action_arguments, local_path, remote_path, + repository, + storage, + local_borg_version, + action_arguments, + local_path, + remote_path, ) command.execute_hook( @@ -635,7 +665,8 @@ def collect_configuration_run_summary_logs(configs, arguments): logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") try: borg_umount.unmount_archive( - mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs), + mount_point=arguments['umount'].mount_point, + local_path=get_local_path(configs), ) except (CalledProcessError, OSError) as error: yield from log_error_records('Error unmounting mount point', error) diff --git a/borgmatic/hooks/cronitor.py b/borgmatic/hooks/cronitor.py index 5fc7c5f3..f514103d 100644 --- a/borgmatic/hooks/cronitor.py +++ b/borgmatic/hooks/cronitor.py @@ -35,9 +35,11 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_ dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' try: - ping_url = f"{hook_config['sss']}/{MONITOR_STATE_TO_CRONITOR[state]}" + ping_url = f"{hook_config[action_name]}/{MONITOR_STATE_TO_CRONITOR[state]}" except KeyError: - logger.debug(f'{config_filename}: Skipping Cronitor {state.name.lower()} ping due to unconfigured action: {action_name}') + logger.debug( + f'{config_filename}: Skipping Cronitor {state.name.lower()} ping due to unconfigured action: {action_name}' + ) return logger.info(f'{config_filename}: Pinging Cronitor {state.name.lower()}{dry_run_label}') -- 2.40.1 From 08e8262ee7731fecefd9e137a72bb3dd5034a059 Mon Sep 17 00:00:00 2001 From: jetchirag Date: Sat, 29 Jul 2023 00:38:49 +0530 Subject: [PATCH 6/6] Minor suggestion fixes Signed-off-by: jetchirag --- borgmatic/commands/arguments.py | 1 - borgmatic/commands/borgmatic.py | 4 +-- tests/unit/commands/test_borgmatic.py | 38 +++++++++++++-------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 11ef33b3..ddff26d8 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -146,7 +146,6 @@ def make_parsers(): help='Go through the motions, but do not actually write to any repositories', ) global_group.add_argument( - '-nm', '--skip-monitoring', dest='skip_monitoring', action='store_true', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c2e7b656..3c2dcc0a 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -66,9 +66,9 @@ def run_configuration(config_filename, config, arguments): error_repository = '' using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) - skip_monitoring = getattr(global_arguments, 'skip_monitoring', False) + skip_monitoring = global_arguments.skip_monitoring action_names = [ - action for action in arguments.keys() if action != 'global' and isinstance(action, str) + action for action in arguments.keys() if action != 'global' ] try: diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 19ac00de..135ba051 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -16,7 +16,7 @@ def test_run_configuration_runs_actions_for_each_repository(): expected_results[1:] ) config = {'location': {'repositories': ['foo', 'bar']}} - arguments = {'global': flexmock(monitoring_verbosity=1)} + arguments = {'global': flexmock(monitoring_verbosity=1, skip_monitoring=False)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -30,7 +30,7 @@ def test_run_configuration_with_invalid_borg_version_errors(): flexmock(module.dispatch).should_receive('call_hooks').never() flexmock(module).should_receive('run_actions').never() config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'prune': flexmock()} list(module.run_configuration('test.yaml', config, arguments)) @@ -45,7 +45,7 @@ def test_run_configuration_logs_monitor_start_error(): flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').never() config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -60,7 +60,7 @@ def test_run_configuration_bails_for_monitor_start_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('run_actions').never() config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -76,7 +76,7 @@ def test_run_configuration_logs_actions_error(): flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False)} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -92,7 +92,7 @@ def test_run_configuration_bails_for_actions_soft_failure(): flexmock(module).should_receive('log_error_records').never() flexmock(module.command).should_receive('considered_soft_failure').and_return(True) config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -109,7 +109,7 @@ def test_run_configuration_logs_monitor_log_error(): flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -127,7 +127,7 @@ def test_run_configuration_bails_for_monitor_log_soft_failure(): flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -144,7 +144,7 @@ def test_run_configuration_logs_monitor_finish_error(): flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_return([]) config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -162,7 +162,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -179,7 +179,7 @@ def test_run_configuration_logs_on_error_hook_error(): ).and_return(expected_results[1:]) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -195,7 +195,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'location': {'repositories': ['foo']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) @@ -210,7 +210,7 @@ def test_run_configuration_retries_soft_error(): flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once() config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == [] @@ -232,7 +232,7 @@ def test_run_configuration_retries_hard_error(): 'foo: Error running actions for repository', OSError, ).and_return(error_logs) config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -250,7 +250,7 @@ def test_run_configuration_repos_ordered(): 'bar: Error running actions for repository', OSError ).and_return(expected_results[1:]).ordered() config = {'location': {'repositories': ['foo', 'bar']}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results @@ -281,7 +281,7 @@ def test_run_configuration_retries_round_robbin(): 'bar: Error running actions for repository', OSError ).and_return(bar_error_logs).ordered() config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == foo_error_logs + bar_error_logs @@ -310,7 +310,7 @@ def test_run_configuration_retries_one_passes(): 'bar: Error running actions for repository', OSError ).and_return(error_logs).ordered() config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -349,7 +349,7 @@ def test_run_configuration_retry_wait(): 'foo: Error running actions for repository', OSError ).and_return(error_logs).ordered() config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs @@ -387,7 +387,7 @@ def test_run_configuration_retries_timeout_multiple_repos(): 'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1, 'retry_wait': 10}, } - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, skip_monitoring=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == error_logs -- 2.40.1