From 38ebfd29698a84e07a87faab4541f108059ea1e9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 15 Nov 2021 11:51:17 -0800 Subject: [PATCH] Rename retry_timeout to retry_wait and standardize log formatting (#28). --- NEWS | 1 + borgmatic/commands/borgmatic.py | 10 ++++++---- borgmatic/config/schema.yaml | 11 ++++++----- tests/unit/commands/test_borgmatic.py | 16 ++++++++-------- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/NEWS b/NEWS index 51e4a9c4..5ca8fa9f 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 1.5.21.dev0 + * #28: Optionally retry failing backups via "retries" and "retry_wait" configuration options. * #459: Add support for old version (2.x) of jsonschema library. 1.5.20 diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 1bf601cb..255ee833 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -55,7 +55,7 @@ def run_configuration(config_filename, config, arguments): local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') retries = storage.get('retries', 0) - retry_timeout = storage.get('retry_timeout', 0) + retry_wait = storage.get('retry_wait', 0) borg_environment.initialize(storage) encountered_error = None error_repository = '' @@ -130,9 +130,9 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository_path, retry_num = repo_queue.get() - timeout = retry_num * retry_timeout + timeout = retry_num * retry_wait if timeout: - logger.warning(f'Sleeping {timeout}s before next retry') + logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') time.sleep(timeout) try: yield from run_actions( @@ -152,7 +152,9 @@ def run_configuration(config_filename, config, arguments): ) if retry_num < retries: repo_queue.put((repository_path, retry_num + 1),) - logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}') + logger.warning( + f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}' + ) continue encountered_error = error error_repository = repository_path diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 0cc90748..f3fc0db0 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -254,14 +254,15 @@ properties: retries: type: integer description: | - Number of times to retry a backup before failing. Defaults - to 0 (i.e. does not attempt retry). + Number of times to retry a failing backup before giving up. + Defaults to 0 (i.e., does not attempt retry). example: 3 - retry_timeout: + retry_wait: type: integer description: | - Wait time between retries, to allow transient issues to pass - Defaults to 0s. + Wait time between retries (in seconds) to allow transient + issues to pass. Increases after each retry as a form of + backoff. Defaults to 0 (no wait). example: 10 temporary_directory: type: string diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index bb81a94d..83db9c2e 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -185,7 +185,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): assert results == expected_results -def test_run_retries_soft_error(): +def test_run_configuration_retries_soft_error(): # Run action first fails, second passes flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') @@ -198,7 +198,7 @@ def test_run_retries_soft_error(): assert results == expected_results -def test_run_retries_hard_error(): +def test_run_configuration_retries_hard_error(): # Run action fails twice flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') @@ -234,7 +234,7 @@ def test_run_repos_ordered(): assert results == expected_results -def test_run_retries_round_robbin(): +def test_run_configuration_retries_round_robbin(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) @@ -257,7 +257,7 @@ def test_run_retries_round_robbin(): assert results == expected_results -def test_run_retries_one_passes(): +def test_run_configuration_retries_one_passes(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( @@ -279,7 +279,7 @@ def test_run_retries_one_passes(): assert results == expected_results -def test_run_retry_timeout(): +def test_run_configuration_retry_wait(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) @@ -302,13 +302,13 @@ def test_run_retry_timeout(): flexmock(module).should_receive('make_error_log_records').with_args( 'foo: Error running actions for repository', OSError ).and_return(expected_results[3:4]).ordered() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_timeout': 10}} + config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results -def test_run_retries_timeout_multiple_repos(): +def test_run_configuration_retries_timeout_multiple_repos(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( @@ -332,7 +332,7 @@ def test_run_retries_timeout_multiple_repos(): ).and_return(expected_results[2:3]).ordered() config = { 'location': {'repositories': ['foo', 'bar']}, - 'storage': {'retries': 1, 'retry_timeout': 10}, + 'storage': {'retries': 1, 'retry_wait': 10}, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments))