From 4cc4b8d4844604291cf3c62ee0c72f524997aa67 Mon Sep 17 00:00:00 2001 From: cadamswaite Date: Wed, 14 Jul 2021 22:46:02 +0100 Subject: [PATCH 1/7] Add queue based retry logic --- borgmatic/commands/borgmatic.py | 17 ++++++++++++++--- borgmatic/config/schema.yaml | 6 ++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 1c3de4524..c92dac752 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -5,6 +5,7 @@ import logging import os import sys from subprocess import CalledProcessError +from queue import Queue import colorama import pkg_resources @@ -52,6 +53,7 @@ def run_configuration(config_filename, config, arguments): local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') + retries = storage.get('retries',1) borg_environment.initialize(storage) encountered_error = None error_repository = '' @@ -120,7 +122,12 @@ def run_configuration(config_filename, config, arguments): ) if not encountered_error: - for repository_path in location['repositories']: + repo_queue = Queue() + for repo in location['repositories']: + repo_queue.put((repo,0),) + + while not repo_queue.empty(): + repository_path, retry_num = repo_queue.get() try: yield from run_actions( arguments=arguments, @@ -134,11 +141,15 @@ def run_configuration(config_filename, config, arguments): repository_path=repository_path, ) except (OSError, CalledProcessError, ValueError) as error: - encountered_error = error - error_repository = repository_path yield from make_error_log_records( '{}: Error running actions for repository'.format(repository_path), error ) + if retry_num < retries: + repo_queue.put((repository_path,retry_num + 1),) + logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}') + continue + encountered_error = error + error_repository = repository_path if not encountered_error: try: diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 00df52e33..eb8bea604 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -249,6 +249,12 @@ properties: Remote network upload rate limit in kiBytes/second. Defaults to unlimited. example: 100 + retries: + type: integer + description: | + Number of times to retry a backup before failing. Defaults + to 1. + example: 3 temporary_directory: type: string description: | From 510449ce65d158b8f200d4022ee181e25462828b Mon Sep 17 00:00:00 2001 From: cadamswaite Date: Wed, 14 Jul 2021 22:49:03 +0100 Subject: [PATCH 2/7] Change default retries to 0 --- borgmatic/commands/borgmatic.py | 2 +- borgmatic/config/schema.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c92dac752..f490d8984 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -53,7 +53,7 @@ def run_configuration(config_filename, config, arguments): local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') - retries = storage.get('retries',1) + retries = storage.get('retries', 0) borg_environment.initialize(storage) encountered_error = None error_repository = '' diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index eb8bea604..fc043f224 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -253,7 +253,7 @@ properties: type: integer description: | Number of times to retry a backup before failing. Defaults - to 1. + to 0 (i.e. does not attempt retry). example: 3 temporary_directory: type: string From 4f36fe2b9ff392f8c55cb4d0bbb77fade67c9a2e Mon Sep 17 00:00:00 2001 From: cadamswaite Date: Wed, 14 Jul 2021 22:53:01 +0100 Subject: [PATCH 3/7] Run Black on changed file --- borgmatic/commands/borgmatic.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index f490d8984..48a9df648 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -124,8 +124,10 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: repo_queue = Queue() for repo in location['repositories']: - repo_queue.put((repo,0),) - + repo_queue.put( + (repo, 0), + ) + while not repo_queue.empty(): repository_path, retry_num = repo_queue.get() try: @@ -145,7 +147,9 @@ def run_configuration(config_filename, config, arguments): '{}: Error running actions for repository'.format(repository_path), error ) if retry_num < retries: - repo_queue.put((repository_path,retry_num + 1),) + repo_queue.put( + (repository_path, retry_num + 1), + ) logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}') continue encountered_error = error @@ -268,7 +272,7 @@ def run_actions( hooks, local_path, remote_path, - repository_path + repository_path, ): # pragma: no cover ''' Given parsed command-line arguments as an argparse.ArgumentParser instance, several different From 89baf757cf93650a942f0d2bf6fd155487b0b965 Mon Sep 17 00:00:00 2001 From: cadamswaite Date: Wed, 14 Jul 2021 23:17:35 +0100 Subject: [PATCH 4/7] Sort imports --- borgmatic/commands/borgmatic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 48a9df648..abecd4e3b 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -4,8 +4,8 @@ import json import logging import os import sys -from subprocess import CalledProcessError from queue import Queue +from subprocess import CalledProcessError import colorama import pkg_resources From 19cad899786ce9b9e87f1dbc2ba4f597f2406e7f Mon Sep 17 00:00:00 2001 From: cadamswaite Date: Sun, 14 Nov 2021 21:35:23 +0000 Subject: [PATCH 5/7] Add some tests for retry logic --- tests/unit/commands/test_borgmatic.py | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 95947f68c..eef2d3a94 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -183,6 +183,79 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): assert results == expected_results +def test_run_retries_soft_error(): + # Run action first fails, second passes + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) + expected_results = [flexmock()] + flexmock(module).should_receive('make_error_log_records').and_return(expected_results).once() + config = {'location': {'repositories': ['foo']}, 'storage': {'retries':1}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == expected_results + +def test_run_retries_hard_error(): + # Run action fails twice + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) + expected_results = [flexmock(), flexmock()] + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]) \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:]).twice() + config = {'location': {'repositories': ['foo']}, 'storage': {'retries':1}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == expected_results + +def test_run_repos_ordered(): + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) + expected_results = [flexmock(), flexmock()] + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]).ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:]).ordered() + config = {'location': {'repositories': ['foo','bar']}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == expected_results + +def test_run_retries_round_robbin(): + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) + expected_results = [flexmock(), flexmock(), flexmock(), flexmock()] + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered() + config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == expected_results + +def test_run_retries_one_passes(): + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4) + expected_results = [flexmock(), flexmock(), flexmock()] + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() + config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == expected_results def test_load_configurations_collects_parsed_configurations(): configuration = flexmock() From b4117916b874c4c3d596d19cb86ce3f4636c329b Mon Sep 17 00:00:00 2001 From: cadamswaite Date: Sun, 14 Nov 2021 22:15:22 +0000 Subject: [PATCH 6/7] Add timeout and tests --- borgmatic/commands/borgmatic.py | 6 ++++ borgmatic/config/schema.yaml | 6 ++++ tests/unit/commands/test_borgmatic.py | 47 +++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index abecd4e3b..b3796c6bf 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -4,6 +4,7 @@ import json import logging import os import sys +import time from queue import Queue from subprocess import CalledProcessError @@ -54,6 +55,7 @@ def run_configuration(config_filename, config, arguments): local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') retries = storage.get('retries', 0) + retry_timeout = storage.get('retry_timeout', 0) borg_environment.initialize(storage) encountered_error = None error_repository = '' @@ -130,6 +132,10 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository_path, retry_num = repo_queue.get() + timeout = retry_num * retry_timeout + if timeout: + logger.warning(f'Sleeping {timeout}s before next retry') + time.sleep(timeout) try: yield from run_actions( arguments=arguments, diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index b30ae0c8b..ae92f9b22 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -257,6 +257,12 @@ properties: Number of times to retry a backup before failing. Defaults to 0 (i.e. does not attempt retry). example: 3 + retry_timeout: + type: integer + description: | + Wait time between retries, to allow transient issues to pass. + Defaults to 0s. + example: 10 temporary_directory: type: string description: | diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index eef2d3a94..5e67ee9a6 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1,5 +1,6 @@ import logging import subprocess +import time from flexmock import flexmock @@ -257,6 +258,52 @@ def test_run_retries_one_passes(): results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results +def test_run_retry_timeout(): + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) + expected_results = [flexmock(), flexmock(), flexmock(), flexmock()] + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() + + flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + + flexmock(time).should_receive('sleep').with_args(20).and_return().ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() + + flexmock(time).should_receive('sleep').with_args(30).and_return().ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered() + config = {'location': {'repositories': ['foo']}, 'storage': {'retries':3, 'retry_timeout':10}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == expected_results + +def test_run_retries_timeout_multiple_repos(): + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4) + expected_results = [flexmock(), flexmock(), flexmock()] + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + + # Sleep before retrying foo (and passing) + flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() + + # Sleep before retrying bar (and failing) + flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() + config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1, 'retry_timeout':10}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == expected_results + def test_load_configurations_collects_parsed_configurations(): configuration = flexmock() other_configuration = flexmock() From 976a877a258d0a00477319a0d8f7bcf7772169d9 Mon Sep 17 00:00:00 2001 From: cadamswaite Date: Sun, 14 Nov 2021 22:37:42 +0000 Subject: [PATCH 7/7] Formatting --- borgmatic/commands/borgmatic.py | 8 +- borgmatic/config/schema.yaml | 2 +- tests/unit/commands/test_borgmatic.py | 125 ++++++++++++++++---------- 3 files changed, 83 insertions(+), 52 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b3796c6bf..1bf601cb5 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -126,9 +126,7 @@ def run_configuration(config_filename, config, arguments): if not encountered_error: repo_queue = Queue() for repo in location['repositories']: - repo_queue.put( - (repo, 0), - ) + repo_queue.put((repo, 0),) while not repo_queue.empty(): repository_path, retry_num = repo_queue.get() @@ -153,9 +151,7 @@ def run_configuration(config_filename, config, arguments): '{}: Error running actions for repository'.format(repository_path), error ) if retry_num < retries: - repo_queue.put( - (repository_path, retry_num + 1), - ) + repo_queue.put((repository_path, retry_num + 1),) logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}') continue encountered_error = error diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index ae92f9b22..0cc907487 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -260,7 +260,7 @@ properties: retry_timeout: type: integer description: | - Wait time between retries, to allow transient issues to pass. + Wait time between retries, to allow transient issues to pass Defaults to 0s. example: 10 temporary_directory: diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 5e67ee9a6..bb81a94d5 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -184,6 +184,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): assert results == expected_results + def test_run_retries_soft_error(): # Run action first fails, second passes flexmock(module.borg_environment).should_receive('initialize') @@ -191,119 +192,153 @@ def test_run_retries_soft_error(): flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) expected_results = [flexmock()] flexmock(module).should_receive('make_error_log_records').and_return(expected_results).once() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries':1}} + config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retries_hard_error(): # Run action fails twice flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) expected_results = [flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]) \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:]).twice() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries':1}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[:1]).with_args( + 'foo: Error running actions for repository', OSError + ).and_return( + expected_results[1:] + ).twice() + config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_repos_ordered(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) expected_results = [flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:]).ordered() - config = {'location': {'repositories': ['foo','bar']}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[1:]).ordered() + config = {'location': {'repositories': ['foo', 'bar']}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retries_round_robbin(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) expected_results = [flexmock(), flexmock(), flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered() - config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[2:3]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[3:4]).ordered() + config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retries_one_passes(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') - flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4) + flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( + [] + ).and_raise(OSError).times(4) expected_results = [flexmock(), flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() - config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[2:3]).ordered() + config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retry_timeout(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) expected_results = [flexmock(), flexmock(), flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[0:1]).ordered() flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[1:2]).ordered() flexmock(time).should_receive('sleep').with_args(20).and_return().ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[2:3]).ordered() flexmock(time).should_receive('sleep').with_args(30).and_return().ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered() - config = {'location': {'repositories': ['foo']}, 'storage': {'retries':3, 'retry_timeout':10}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[3:4]).ordered() + config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_timeout': 10}} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_run_retries_timeout_multiple_repos(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook') - flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4) + flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( + [] + ).and_raise(OSError).times(4) expected_results = [flexmock(), flexmock(), flexmock()] - flexmock(module).should_receive('make_error_log_records') \ - .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'foo: Error running actions for repository', OSError + ).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[1:2]).ordered() # Sleep before retrying foo (and passing) flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() - + # Sleep before retrying bar (and failing) flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() - flexmock(module).should_receive('make_error_log_records') \ - .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() - config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1, 'retry_timeout':10}} + flexmock(module).should_receive('make_error_log_records').with_args( + 'bar: Error running actions for repository', OSError + ).and_return(expected_results[2:3]).ordered() + config = { + 'location': {'repositories': ['foo', 'bar']}, + 'storage': {'retries': 1, 'retry_timeout': 10}, + } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results + def test_load_configurations_collects_parsed_configurations(): configuration = flexmock() other_configuration = flexmock()