From dbf8301c19fdd16aace7a4a9ac52886eba3e0fbc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Feb 2023 10:47:17 -0800 Subject: [PATCH] Add "checkpoint_volume" configuration option to creates checkpoints every specified number of bytes. --- NEWS | 2 ++ borgmatic/borg/create.py | 2 ++ borgmatic/config/schema.yaml | 10 +++++++++ tests/unit/borg/test_create.py | 41 ++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/NEWS b/NEWS index 8e8e1de..3a0b3e8 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,8 @@ * Add "--repository" flag to the "rcreate" action to optionally select one configured repository to create. * Add "--progress" flag to the "transfer" action, new in Borg 2.0.0b5. + * Add "checkpoint_volume" configuration option to creates checkpoints every specified number of + bytes during a long-running backup, new in Borg 2.0.0b5. 1.7.7 * #642: Add MySQL database hook "add_drop_database" configuration option to control whether dumped diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 39676e6..5dde583 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -337,6 +337,7 @@ def create_archive( expand_home_directories(location_config.get('exclude_patterns')) ) checkpoint_interval = storage_config.get('checkpoint_interval', None) + checkpoint_volume = storage_config.get('checkpoint_volume', None) chunker_params = storage_config.get('chunker_params', None) compression = storage_config.get('compression', None) upload_rate_limit = storage_config.get('upload_rate_limit', None) @@ -381,6 +382,7 @@ def create_archive( + make_pattern_flags(location_config, pattern_file.name if pattern_file else None) + make_exclude_flags(location_config, exclude_file.name if exclude_file else None) + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ()) + + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ()) + (('--chunker-params', chunker_params) if chunker_params else ()) + (('--compression', compression) if compression else ()) + upload_ratelimit_flags diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d7acc1c..4a1c8e0 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -240,6 +240,16 @@ properties: for details. Defaults to checkpoints every 1800 seconds (30 minutes). example: 1800 + checkpoint_volume: + type: integer + description: | + Number of backed up bytes between each checkpoint during a + long-running backup. Only supported with Borg 2+. See + https://borgbackup.readthedocs.io/en/stable/faq.html + for details. Defaults to only time-based checkpointing (see + "checkpoint_interval") instead of volume-based + checkpointing. + example: 1048576 chunker_params: type: string description: | diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 24d5c3e..ecf455b 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -874,6 +874,47 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte ) +def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('map_directories_to_devices').and_return({}) + flexmock(module).should_receive('expand_directories').and_return(()) + flexmock(module).should_receive('pattern_root_directories').and_return([]) + flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) + flexmock(module).should_receive('expand_home_directories').and_return(()) + flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module).should_receive('ensure_files_readable') + flexmock(module).should_receive('make_pattern_flags').and_return(()) + flexmock(module).should_receive('make_exclude_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'create', '--checkpoint-volume', '1024') + REPO_ARCHIVE_WITH_PATHS, + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=None, + ) + + module.create_archive( + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'checkpoint_volume': 1024}, + local_borg_version='1.2.3', + ) + + def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER