From 2d3f5fa05d47ee88d8e1f7817ff62ec69d89fad8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 19 Feb 2018 15:51:04 -0800 Subject: [PATCH] Support for Borg --lock-wait option for the maximum wait for a repository/cache lock (#56). --- NEWS | 6 +++-- borgmatic/borg/check.py | 15 ++++++++---- borgmatic/borg/create.py | 4 +++- borgmatic/borg/extract.py | 7 +++--- borgmatic/borg/prune.py | 12 ++++++---- borgmatic/commands/borgmatic.py | 2 ++ borgmatic/config/schema.yaml | 5 ++++ borgmatic/tests/unit/borg/test_check.py | 29 +++++++++++++++++++++++ borgmatic/tests/unit/borg/test_create.py | 20 ++++++++++++++++ borgmatic/tests/unit/borg/test_extract.py | 23 ++++++++++++++++++ borgmatic/tests/unit/borg/test_prune.py | 23 ++++++++++++++++++ setup.py | 2 +- 12 files changed, 132 insertions(+), 16 deletions(-) diff --git a/NEWS b/NEWS index 1cdd1ee..ee65b06 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,9 @@ -1.1.15.dev0 +1.1.15 * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file. * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together. - * #55: Fix for missing tags/releases from Gitea and GitHub project hosting. + Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298 + * #55: Fix for missing tags/releases on Gitea and GitHub project hosting. + * #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock. * #58: Support for using tilde in exclude_patterns to reference home directory. 1.1.14 diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 41a353c..04b2daa 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -60,18 +60,23 @@ def _make_check_flags(checks, check_last=None): ) + last_flag -def check_archives(verbosity, repository, consistency_config, local_path='borg', remote_path=None): +def check_archives(verbosity, repository, storage_config, consistency_config, local_path='borg', + remote_path=None): ''' - Given a verbosity flag, a local or remote repository path, a consistency config dict, and a - local/remote commands to run, check the contained Borg archives for consistency. + Given a verbosity flag, a local or remote repository path, a storage config dict, a consistency + config dict, and a local/remote commands to run, check the contained Borg archives for + consistency. If there are no consistency checks to run, skip running them. ''' checks = _parse_checks(consistency_config) check_last = consistency_config.get('check_last', None) + lock_wait = None if set(checks).intersection(set(DEFAULT_CHECKS)): remote_path_flags = ('--remote-path', remote_path) if remote_path else () + lock_wait = storage_config.get('lock_wait', None) + lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else () verbosity_flags = { VERBOSITY_SOME: ('--info',), VERBOSITY_LOTS: ('--debug',), @@ -80,7 +85,7 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg', full_command = ( local_path, 'check', repository, - ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags + ) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags # The check command spews to stdout/stderr even without the verbose flag. Suppress it. stdout = None if verbosity_flags else open(os.devnull, 'w') @@ -89,4 +94,4 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg', subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) if 'extract' in checks: - extract.extract_last_archive_dry_run(verbosity, repository, local_path, remote_path) + extract.extract_last_archive_dry_run(verbosity, repository, lock_wait, local_path, remote_path) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 22c6c0b..bbc9308 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -129,6 +129,8 @@ def create_archive( remote_rate_limit_flags = ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else () umask = storage_config.get('umask', None) umask_flags = ('--umask', str(umask)) if umask else () + lock_wait = storage_config.get('lock_wait', None) + lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else () one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else () files_cache = location_config.get('files_cache') files_cache_flags = ('--files-cache', files_cache) if files_cache else () @@ -149,7 +151,7 @@ def create_archive( ), ) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \ one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \ - verbosity_flags + dry_run_flags + lock_wait_flags + verbosity_flags + dry_run_flags logger.debug(' '.join(full_command)) subprocess.check_call(full_command) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 3d722ad..d3d62a3 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -8,12 +8,13 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS logger = logging.getLogger(__name__) -def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remote_path=None): +def extract_last_archive_dry_run(verbosity, repository, lock_wait=None, local_path='borg', remote_path=None): ''' Perform an extraction dry-run of just the most recent archive. If there are no archives, skip the dry-run. ''' remote_path_flags = ('--remote-path', remote_path) if remote_path else () + lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else () verbosity_flags = { VERBOSITY_SOME: ('--info',), VERBOSITY_LOTS: ('--debug',), @@ -23,7 +24,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot local_path, 'list', '--short', repository, - ) + remote_path_flags + verbosity_flags + ) + remote_path_flags + lock_wait_flags + verbosity_flags list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding) @@ -39,7 +40,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot repository=repository, last_archive_name=last_archive_name, ), - ) + remote_path_flags + verbosity_flags + list_flag + ) + remote_path_flags + lock_wait_flags + verbosity_flags + list_flag logger.debug(' '.join(full_extract_command)) subprocess.check_call(full_extract_command) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index dc66d38..3054d81 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -32,12 +32,16 @@ def _make_prune_flags(retention_config): ) -def prune_archives(verbosity, dry_run, repository, retention_config, local_path='borg', remote_path=None): +def prune_archives(verbosity, dry_run, repository, storage_config, retention_config, + local_path='borg', remote_path=None): ''' - Given verbosity/dry-run flags, a local or remote repository path, a retention config dict, prune - Borg archives according the the retention policy specified in that configuration. + Given verbosity/dry-run flags, a local or remote repository path, a storage config dict, and a + retention config dict, prune Borg archives according the the retention policy specified in that + configuration. ''' remote_path_flags = ('--remote-path', remote_path) if remote_path else () + lock_wait = storage_config.get('lock_wait', None) + lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else () verbosity_flags = { VERBOSITY_SOME: ('--info', '--stats',), VERBOSITY_LOTS: ('--debug', '--stats', '--list'), @@ -51,7 +55,7 @@ def prune_archives(verbosity, dry_run, repository, retention_config, local_path= element for pair in _make_prune_flags(retention_config) for element in pair - ) + remote_path_flags + verbosity_flags + dry_run_flags + ) + remote_path_flags + lock_wait_flags + verbosity_flags + dry_run_flags logger.debug(' '.join(full_command)) subprocess.check_call(full_command) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index da53040..e2bffee 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -113,6 +113,7 @@ def run_configuration(config_filename, args): # pragma: no cover args.verbosity, args.dry_run, repository, + storage, retention, local_path=local_path, remote_path=remote_path, @@ -133,6 +134,7 @@ def run_configuration(config_filename, args): # pragma: no cover check.check_archives( args.verbosity, repository, + storage, consistency, local_path=local_path, remote_path=remote_path, diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index b91ce5d..791e3c1 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -139,6 +139,10 @@ map: type: scalar desc: Umask to be used for borg create. example: 0077 + lock_wait: + type: int + desc: Maximum seconds to wait for acquiring a repository/cache lock. + example: 5 archive_name_format: type: scalar desc: | @@ -152,6 +156,7 @@ map: desc: | Retention policy for how many backups to keep in each category. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. + At least one of the "keep" options is required for pruning to work. map: keep_within: type: scalar diff --git a/borgmatic/tests/unit/borg/test_check.py b/borgmatic/tests/unit/borg/test_check.py index 52b1b62..361a720 100644 --- a/borgmatic/tests/unit/borg/test_check.py +++ b/borgmatic/tests/unit/borg/test_check.py @@ -103,6 +103,7 @@ def test_check_archives_calls_borg_with_parameters(checks): module.check_archives( verbosity=None, repository='repo', + storage_config={}, consistency_config=consistency_config, ) @@ -119,6 +120,7 @@ def test_check_archives_with_extract_check_calls_extract_only(): module.check_archives( verbosity=None, repository='repo', + storage_config={}, consistency_config=consistency_config, ) @@ -136,6 +138,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter(): module.check_archives( verbosity=VERBOSITY_SOME, repository='repo', + storage_config={}, consistency_config=consistency_config, ) @@ -153,6 +156,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter(): module.check_archives( verbosity=VERBOSITY_LOTS, repository='repo', + storage_config={}, consistency_config=consistency_config, ) @@ -165,6 +169,7 @@ def test_check_archives_without_any_checks_bails(): module.check_archives( verbosity=None, repository='repo', + storage_config={}, consistency_config=consistency_config, ) @@ -186,6 +191,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): module.check_archives( verbosity=None, repository='repo', + storage_config={}, consistency_config=consistency_config, local_path='borg1', ) @@ -208,6 +214,29 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( module.check_archives( verbosity=None, repository='repo', + storage_config={}, consistency_config=consistency_config, remote_path='borg1', ) + + +def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): + checks = ('repository',) + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) + stdout = flexmock() + insert_subprocess_mock( + ('borg', 'check', 'repo', '--lock-wait', '5'), + stdout=stdout, stderr=STDOUT, + ) + flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) + flexmock(module.os).should_receive('devnull') + + module.check_archives( + verbosity=None, + repository='repo', + storage_config={'lock_wait': 5}, + consistency_config=consistency_config, + ) diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index c0a5405..8831e63 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -518,6 +518,26 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): ) +def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) + insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5')) + + module.create_archive( + verbosity=None, + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'lock_wait': 5}, + ) + + def test_create_archive_with_source_directories_glob_expands(): flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) diff --git a/borgmatic/tests/unit/borg/test_extract.py b/borgmatic/tests/unit/borg/test_extract.py index 194e33c..9811a78 100644 --- a/borgmatic/tests/unit/borg/test_extract.py +++ b/borgmatic/tests/unit/borg/test_extract.py @@ -34,6 +34,7 @@ def test_extract_last_archive_dry_run_should_call_borg_with_last_archive(): module.extract_last_archive_dry_run( verbosity=None, repository='repo', + lock_wait=None, ) @@ -48,6 +49,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_bail(): module.extract_last_archive_dry_run( verbosity=None, repository='repo', + lock_wait=None, ) @@ -64,6 +66,7 @@ def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_ module.extract_last_archive_dry_run( verbosity=VERBOSITY_SOME, repository='repo', + lock_wait=None, ) @@ -80,6 +83,7 @@ def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_ module.extract_last_archive_dry_run( verbosity=VERBOSITY_LOTS, repository='repo', + lock_wait=None, ) @@ -96,6 +100,7 @@ def test_extract_last_archive_dry_run_should_call_borg_via_local_path(): module.extract_last_archive_dry_run( verbosity=None, repository='repo', + lock_wait=None, local_path='borg1', ) @@ -113,5 +118,23 @@ def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_paramete module.extract_last_archive_dry_run( verbosity=None, repository='repo', + lock_wait=None, remote_path='borg1', ) + + +def test_extract_last_archive_dry_run_should_call_borg_with_lock_wait_parameters(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--lock-wait', '5'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5'), + ) + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + lock_wait=5, + ) diff --git a/borgmatic/tests/unit/borg/test_prune.py b/borgmatic/tests/unit/borg/test_prune.py index 31fe937..9289d0b 100644 --- a/borgmatic/tests/unit/borg/test_prune.py +++ b/borgmatic/tests/unit/borg/test_prune.py @@ -66,6 +66,7 @@ def test_prune_archives_calls_borg_with_parameters(): verbosity=None, dry_run=False, repository='repo', + storage_config={}, retention_config=retention_config, ) @@ -79,6 +80,7 @@ def test_prune_archives_with_verbosity_some_calls_borg_with_info_parameter(): module.prune_archives( repository='repo', + storage_config={}, verbosity=VERBOSITY_SOME, dry_run=False, retention_config=retention_config, @@ -94,6 +96,7 @@ def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter(): module.prune_archives( repository='repo', + storage_config={}, verbosity=VERBOSITY_LOTS, dry_run=False, retention_config=retention_config, @@ -109,6 +112,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): module.prune_archives( repository='repo', + storage_config={}, verbosity=None, dry_run=True, retention_config=retention_config, @@ -126,6 +130,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): verbosity=None, dry_run=False, repository='repo', + storage_config={}, retention_config=retention_config, local_path='borg1', ) @@ -142,6 +147,24 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters( verbosity=None, dry_run=False, repository='repo', + storage_config={}, retention_config=retention_config, remote_path='borg1', ) + + +def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): + storage_config = {'lock_wait': 5} + retention_config = flexmock() + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) + insert_subprocess_mock(PRUNE_COMMAND + ('--lock-wait', '5')) + + module.prune_archives( + verbosity=None, + dry_run=False, + repository='repo', + storage_config=storage_config, + retention_config=retention_config, + ) diff --git a/setup.py b/setup.py index c632362..82e22ab 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.15.dev' +VERSION = '1.1.15' setup(