From ca4312bb852bd67f3a820251ae64f23cee83e6ac Mon Sep 17 00:00:00 2001 From: Dan Date: Thu, 2 Nov 2017 22:22:40 -0700 Subject: [PATCH] Support for Borg --remote-ratelimit for limiting upload rate. And log Borg commands. --- NEWS | 2 ++ borgmatic/borg/check.py | 5 +++++ borgmatic/borg/create.py | 12 ++++++++++-- borgmatic/borg/extract.py | 5 +++++ borgmatic/borg/prune.py | 5 +++++ borgmatic/config/schema.yaml | 11 ++++++++--- borgmatic/tests/unit/borg/test_create.py | 18 ++++++++++++++++++ 7 files changed, 53 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index bd5998a7..7be76e36 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,8 @@ shuts down if borgmatic is terminated (e.g. due to a system suspend). * #29: Support for using tilde in repository paths to reference home directory. * #42: Support for Borg --files-cache option for setting the files cache operation mode. + * #44: Support for Borg --remote-ratelimit for limiting upload rate. + * Log invoked Borg commands when at highest verbosity level. 1.1.9 * #16, #38: Support for user-defined hooks before/after backup, or on error. diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 49805547..e9afc33f 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -1,3 +1,4 @@ +import logging import os import subprocess @@ -8,6 +9,9 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS DEFAULT_CHECKS = ('repository', 'archives') +logger = logging.getLogger(__name__) + + def _parse_checks(consistency_config): ''' Given a consistency config with a "checks" list, transform it to a tuple of named checks to run. @@ -79,6 +83,7 @@ def check_archives(verbosity, repository, consistency_config, remote_path=None): # The check command spews to stdout/stderr even without the verbose flag. Suppress it. stdout = None if verbosity_flags else open(os.devnull, 'w') + logger.debug(' '.join(full_command)) subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) if 'extract' in checks: diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index cdabd560..6f0d7246 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -1,5 +1,6 @@ import glob import itertools +import logging import os import subprocess import tempfile @@ -7,6 +8,9 @@ import tempfile from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS +logger = logging.getLogger(__name__) + + def initialize(storage_config): passphrase = storage_config.get('encryption_passphrase') @@ -81,6 +85,8 @@ def create_archive( ) compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () + remote_rate_limit = storage_config.get('remote_rate_limit', None) + remote_rate_limit_flags = ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else () umask = storage_config.get('umask', None) umask_flags = ('--umask', str(umask)) if umask else () one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else () @@ -101,7 +107,9 @@ def create_archive( repository=repository, archive_name_format=archive_name_format, ), - ) + sources + exclude_flags + compression_flags + one_file_system_flags + files_cache_flags + \ - remote_path_flags + umask_flags + verbosity_flags + ) + sources + exclude_flags + compression_flags + remote_rate_limit_flags + \ + one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \ + verbosity_flags + logger.debug(' '.join(full_command)) subprocess.check_call(full_command) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index fde1ac59..ee640409 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -1,9 +1,13 @@ +import logging import sys import subprocess from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS +logger = logging.getLogger(__name__) + + def extract_last_archive_dry_run(verbosity, repository, remote_path=None): ''' Perform an extraction dry-run of just the most recent archive. If there are no archives, skip @@ -37,4 +41,5 @@ def extract_last_archive_dry_run(verbosity, repository, remote_path=None): ), ) + remote_path_flags + verbosity_flags + list_flag + logger.debug(' '.join(full_extract_command)) subprocess.check_call(full_extract_command) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 0f20c3d6..918f9840 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -1,8 +1,12 @@ +import logging import subprocess from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS +logger = logging.getLogger(__name__) + + def _make_prune_flags(retention_config): ''' Given a retention config dict mapping from option name to value, tranform it into an iterable of @@ -48,4 +52,5 @@ def prune_archives(verbosity, repository, retention_config, remote_path=None): for element in pair ) + remote_path_flags + verbosity_flags + logger.debug(' '.join(full_command)) subprocess.check_call(full_command) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 4918f771..69a660ad 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -24,9 +24,10 @@ map: example: true files_cache: type: scalar - desc: Mode in which to operate the files cache. See - https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for - details. + desc: | + Mode in which to operate the files cache. See + https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for + details. example: ctime,size,inode remote_path: type: scalar @@ -91,6 +92,10 @@ map: https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details. Defaults to no compression. example: lz4 + remote_rate_limit: + type: int + desc: Remote network upload rate limit in kiBytes/second. + example: 100 umask: type: scalar desc: Umask to be used for borg create. diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index c53e2bdc..045aa521 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -231,6 +231,24 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( ) +def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) + insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100')) + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'remote_rate_limit': 100}, + ) + + def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters(): flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None)