From 0c6c61a272b3a7ba6aeedf9c0be5bbdd522801ab Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 4 Dec 2019 15:48:10 -0800 Subject: [PATCH] Pass extra options directly to particular Borg commands, handy for Borg options that borgmatic does not yet support natively (#235). --- NEWS | 5 ++++ borgmatic/borg/check.py | 2 ++ borgmatic/borg/create.py | 2 ++ borgmatic/borg/init.py | 10 +++++--- borgmatic/borg/prune.py | 2 ++ borgmatic/commands/borgmatic.py | 1 + borgmatic/config/schema.yaml | 23 ++++++++++++++++++ setup.py | 2 +- tests/unit/borg/test_check.py | 14 +++++++++++ tests/unit/borg/test_create.py | 25 +++++++++++++++++++ tests/unit/borg/test_init.py | 43 +++++++++++++++++++++++++-------- tests/unit/borg/test_prune.py | 15 ++++++++++++ 12 files changed, 130 insertions(+), 14 deletions(-) diff --git a/NEWS b/NEWS index dac9e98..09def40 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.4.17 + * #235: Pass extra options directly to particular Borg commands, handy for Borg options that + borgmatic does not yet support natively. Use "extra_borg_options" in the storage configuration + section. + 1.4.16 * #256: Fix for "before_backup" hook not triggering an error when the command contains "borg" and has an exit code of 1. diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 78f1b38..89c5855 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -103,6 +103,7 @@ def check_archives( checks = _parse_checks(consistency_config, only_checks) check_last = consistency_config.get('check_last', None) lock_wait = None + extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))): remote_path_flags = ('--remote-path', remote_path) if remote_path else () @@ -123,6 +124,7 @@ def check_archives( + remote_path_flags + lock_wait_flags + verbosity_flags + + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + (repository,) ) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 879a94a..2be2eb0 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -150,6 +150,7 @@ def create_archive( files_cache = location_config.get('files_cache') default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' archive_name_format = storage_config.get('archive_name_format', default_archive_name_format) + extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '') full_command = ( (local_path, 'create') @@ -185,6 +186,7 @@ def create_archive( + (('--dry-run',) if dry_run else ()) + (('--progress',) if progress else ()) + (('--json',) if json else ()) + + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + ( '{repository}::{archive_name_format}'.format( repository=repository, archive_name_format=archive_name_format diff --git a/borgmatic/borg/init.py b/borgmatic/borg/init.py index 1ec25c0..152f8c1 100644 --- a/borgmatic/borg/init.py +++ b/borgmatic/borg/init.py @@ -11,6 +11,7 @@ INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2 def initialize_repository( repository, + storage_config, encryption_mode, append_only=None, storage_quota=None, @@ -18,9 +19,9 @@ def initialize_repository( remote_path=None, ): ''' - Given a local or remote repository path, a Borg encryption mode, whether the repository should - be append-only, and the storage quota to use, initialize the repository. If the repository - already exists, then log and skip initialization. + Given a local or remote repository path, a storage configuration dict, a Borg encryption mode, + whether the repository should be append-only, and the storage quota to use, initialize the + repository. If the repository already exists, then log and skip initialization. ''' info_command = (local_path, 'info', repository) logger.debug(' '.join(info_command)) @@ -33,6 +34,8 @@ def initialize_repository( if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE: raise + extra_borg_options = storage_config.get('extra_borg_options', {}).get('init', '') + init_command = ( (local_path, 'init') + (('--encryption', encryption_mode) if encryption_mode else ()) @@ -41,6 +44,7 @@ def initialize_repository( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + (('--remote-path', remote_path) if remote_path else ()) + + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + (repository,) ) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 584973e..0913cde 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -49,6 +49,7 @@ def prune_archives( ''' umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) + extra_borg_options = storage_config.get('extra_borg_options', {}).get('prune', '') full_command = ( (local_path, 'prune') @@ -61,6 +62,7 @@ def prune_archives( + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (('--stats',) if stats else ()) + + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + (repository,) ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 4cd63bb..c33b248 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -189,6 +189,7 @@ def run_actions( logger.info('{}: Initializing repository'.format(repository)) borg_init.initialize_repository( repository, + storage, arguments['init'].encryption_mode, arguments['init'].append_only, arguments['init'].storage_quota, diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 8b3667f..169b643 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -245,6 +245,29 @@ map: Bypass Borg error about a previously unknown unencrypted repository. Defaults to false. example: true + extra_borg_options: + map: + init: + type: str + desc: Extra command-line options to pass to "borg init". + example: "--make-parent-dirs" + prune: + type: str + desc: Extra command-line options to pass to "borg prune". + example: "--save-space" + create: + type: str + desc: Extra command-line options to pass to "borg create". + example: "--no-files-cache" + check: + type: str + desc: Extra command-line options to pass to "borg check". + example: "--save-space" + desc: | + Additional options to pass directly to particular Borg commands, handy for Borg + options that borgmatic does not yet support natively. Note that borgmatic does + not perform any validation on these options. Running borgmatic with + "--verbosity 2" shows the exact Borg command-line invocation. retention: desc: | Retention policy for how many backups to keep in each category. See diff --git a/setup.py b/setup.py index 46871a9..5f60951 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.4.16' +VERSION = '1.4.17' setup( diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 51d430f..df82936 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -296,3 +296,17 @@ def test_check_archives_with_retention_prefix(): module.check_archives( repository='repo', storage_config={}, consistency_config=consistency_config ) + + +def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): + checks = ('repository',) + consistency_config = {'check_last': None} + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').and_return(()) + insert_execute_command_mock(('borg', 'check', '--extra', '--options', 'repo')) + + module.check_archives( + repository='repo', + storage_config={'extra_borg_options': {'check': '--extra --options'}}, + consistency_config=consistency_config, + ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 5819df0..5a1a3e8 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -1092,3 +1092,28 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, ) + + +def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): + flexmock(module).should_receive('borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_home_directories').and_return(()) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'create', '--extra', '--options') + ARCHIVE_WITH_PATHS, + output_log_level=logging.INFO, + error_on_warnings=False, + ) + + module.create_archive( + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'extra_borg_options': {'create': '--extra --options'}}, + ) diff --git a/tests/unit/borg/test_init.py b/tests/unit/borg/test_init.py index fffdfd7..49e9933 100644 --- a/tests/unit/borg/test_init.py +++ b/tests/unit/borg/test_init.py @@ -32,7 +32,7 @@ def test_initialize_repository_calls_borg_with_parameters(): insert_info_command_not_found_mock() insert_init_command_mock(INIT_COMMAND + ('repo',)) - module.initialize_repository(repository='repo', encryption_mode='repokey') + module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey') def test_initialize_repository_raises_for_borg_init_error(): @@ -42,14 +42,16 @@ def test_initialize_repository_raises_for_borg_init_error(): ) with pytest.raises(subprocess.CalledProcessError): - module.initialize_repository(repository='repo', encryption_mode='repokey') + module.initialize_repository( + repository='repo', storage_config={}, encryption_mode='repokey' + ) def test_initialize_repository_skips_initialization_when_repository_already_exists(): insert_info_command_found_mock() flexmock(module).should_receive('execute_command_without_capture').never() - module.initialize_repository(repository='repo', encryption_mode='repokey') + module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey') def test_initialize_repository_raises_for_unknown_info_command_error(): @@ -58,21 +60,27 @@ def test_initialize_repository_raises_for_unknown_info_command_error(): ) with pytest.raises(subprocess.CalledProcessError): - module.initialize_repository(repository='repo', encryption_mode='repokey') + module.initialize_repository( + repository='repo', storage_config={}, encryption_mode='repokey' + ) def test_initialize_repository_with_append_only_calls_borg_with_append_only_parameter(): insert_info_command_not_found_mock() insert_init_command_mock(INIT_COMMAND + ('--append-only', 'repo')) - module.initialize_repository(repository='repo', encryption_mode='repokey', append_only=True) + module.initialize_repository( + repository='repo', storage_config={}, encryption_mode='repokey', append_only=True + ) def test_initialize_repository_with_storage_quota_calls_borg_with_storage_quota_parameter(): insert_info_command_not_found_mock() insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G', 'repo')) - module.initialize_repository(repository='repo', encryption_mode='repokey', storage_quota='5G') + module.initialize_repository( + repository='repo', storage_config={}, encryption_mode='repokey', storage_quota='5G' + ) def test_initialize_repository_with_log_info_calls_borg_with_info_parameter(): @@ -80,7 +88,7 @@ def test_initialize_repository_with_log_info_calls_borg_with_info_parameter(): insert_init_command_mock(INIT_COMMAND + ('--info', 'repo')) insert_logging_mock(logging.INFO) - module.initialize_repository(repository='repo', encryption_mode='repokey') + module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey') def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter(): @@ -88,18 +96,33 @@ def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter(): insert_init_command_mock(INIT_COMMAND + ('--debug', 'repo')) insert_logging_mock(logging.DEBUG) - module.initialize_repository(repository='repo', encryption_mode='repokey') + module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey') def test_initialize_repository_with_local_path_calls_borg_via_local_path(): insert_info_command_not_found_mock() insert_init_command_mock(('borg1',) + INIT_COMMAND[1:] + ('repo',)) - module.initialize_repository(repository='repo', encryption_mode='repokey', local_path='borg1') + module.initialize_repository( + repository='repo', storage_config={}, encryption_mode='repokey', local_path='borg1' + ) def test_initialize_repository_with_remote_path_calls_borg_with_remote_path_parameter(): insert_info_command_not_found_mock() insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1', 'repo')) - module.initialize_repository(repository='repo', encryption_mode='repokey', remote_path='borg1') + module.initialize_repository( + repository='repo', storage_config={}, encryption_mode='repokey', remote_path='borg1' + ) + + +def test_initialize_repository_with_extra_borg_options_calls_borg_with_extra_options(): + insert_info_command_not_found_mock() + insert_init_command_mock(INIT_COMMAND + ('--extra', '--options', 'repo')) + + module.initialize_repository( + repository='repo', + storage_config={'extra_borg_options': {'init': '--extra --options'}}, + encryption_mode='repokey', + ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index ed1ee67..80cce83 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -188,3 +188,18 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): storage_config=storage_config, retention_config=retention_config, ) + + +def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): + retention_config = flexmock() + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS + ) + insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO) + + module.prune_archives( + dry_run=False, + repository='repo', + storage_config={'extra_borg_options': {'prune': '--extra --options'}}, + retention_config=retention_config, + )