diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index abecd4e3b..b3796c6bf 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -4,6 +4,7 @@ import json import logging import os import sys +import time from queue import Queue from subprocess import CalledProcessError @@ -54,6 +55,7 @@ def run_configuration(config_filename, config, arguments): local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') retries = storage.get('retries', 0) + retry_timeout = storage.get('retry_timeout', 0) borg_environment.initialize(storage) encountered_error = None error_repository = '' @@ -130,6 +132,10 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository_path, retry_num = repo_queue.get() + timeout = retry_num * retry_timeout + if timeout: + logger.warning(f'Sleeping {timeout}s before next retry') + time.sleep(timeout) try: yield from run_actions( arguments=arguments, diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index b30ae0c8b..ae92f9b22 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -257,6 +257,12 @@ properties: Number of times to retry a backup before failing. Defaults to 0 (i.e. does not attempt retry). example: 3 + retry_timeout: + type: integer + description: | + Wait time between retries, to allow transient issues to pass. + Defaults to 0s. + example: 10 temporary_directory: type: string description: | diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index eef2d3a94..5e67ee9a6 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1,5 +1,6 @@ import logging import subprocess +import time from flexmock import flexmock @@ -257,6 +258,52 @@ def test_run_retries_one_passes(): results = list(module.run_configuration('test.yaml', config, arguments)) assert results == expected_results +def test_run_retry_timeout(): + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) + expected_results = [flexmock(), flexmock(), flexmock(), flexmock()] + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() + + flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + + flexmock(time).should_receive('sleep').with_args(20).and_return().ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() + + flexmock(time).should_receive('sleep').with_args(30).and_return().ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered() + config = {'location': {'repositories': ['foo']}, 'storage': {'retries':3, 'retry_timeout':10}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == expected_results + +def test_run_retries_timeout_multiple_repos(): + flexmock(module.borg_environment).should_receive('initialize') + flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4) + expected_results = [flexmock(), flexmock(), flexmock()] + flexmock(module).should_receive('make_error_log_records') \ + .with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() + + # Sleep before retrying foo (and passing) + flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() + + # Sleep before retrying bar (and failing) + flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() + flexmock(module).should_receive('make_error_log_records') \ + .with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() + config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1, 'retry_timeout':10}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, arguments)) + assert results == expected_results + def test_load_configurations_collects_parsed_configurations(): configuration = flexmock() other_configuration = flexmock()