diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 1c3de4524..c92dac752 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -5,6 +5,7 @@ import logging import os import sys from subprocess import CalledProcessError +from queue import Queue import colorama import pkg_resources @@ -52,6 +53,7 @@ def run_configuration(config_filename, config, arguments): local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path') + retries = storage.get('retries',1) borg_environment.initialize(storage) encountered_error = None error_repository = '' @@ -120,7 +122,12 @@ def run_configuration(config_filename, config, arguments): ) if not encountered_error: - for repository_path in location['repositories']: + repo_queue = Queue() + for repo in location['repositories']: + repo_queue.put((repo,0),) + + while not repo_queue.empty(): + repository_path, retry_num = repo_queue.get() try: yield from run_actions( arguments=arguments, @@ -134,11 +141,15 @@ def run_configuration(config_filename, config, arguments): repository_path=repository_path, ) except (OSError, CalledProcessError, ValueError) as error: - encountered_error = error - error_repository = repository_path yield from make_error_log_records( '{}: Error running actions for repository'.format(repository_path), error ) + if retry_num < retries: + repo_queue.put((repository_path,retry_num + 1),) + logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}') + continue + encountered_error = error + error_repository = repository_path if not encountered_error: try: diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 00df52e33..eb8bea604 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -249,6 +249,12 @@ properties: Remote network upload rate limit in kiBytes/second. Defaults to unlimited. example: 100 + retries: + type: integer + description: | + Number of times to retry a backup before failing. Defaults + to 1. + example: 3 temporary_directory: type: string description: |