Support for Borg --remote-ratelimit for limiting upload rate. And log Borg commands.

NEWS

@@ -3,6 +3,8 @@
    shuts down if borgmatic is terminated (e.g. due to a system suspend).
  * #29: Support for using tilde in repository paths to reference home directory.
  * #42: Support for Borg --files-cache option for setting the files cache operation mode.
+ * #44: Support for Borg --remote-ratelimit for limiting upload rate.
+ * Log invoked Borg commands when at highest verbosity level.
 
 1.1.9
  * #16, #38: Support for user-defined hooks before/after backup, or on error.

borgmatic/borg/check.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import subprocess
 
@@ -8,6 +9,9 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 DEFAULT_CHECKS = ('repository', 'archives')
 
 
+logger = logging.getLogger(__name__)
+
+
 def _parse_checks(consistency_config):
     '''
     Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
@@ -79,6 +83,7 @@ def check_archives(verbosity, repository, consistency_config, remote_path=None):
         # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
         stdout = None if verbosity_flags else open(os.devnull, 'w')
 
+        logger.debug(' '.join(full_command))
         subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
 
     if 'extract' in checks:

borgmatic/borg/create.py

@@ -1,5 +1,6 @@
 import glob
 import itertools
+import logging
 import os
 import subprocess
 import tempfile
@@ -7,6 +8,9 @@ import tempfile
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
+logger = logging.getLogger(__name__)
+
+
 def initialize(storage_config):
     passphrase = storage_config.get('encryption_passphrase')
 
@@ -81,6 +85,8 @@ def create_archive(
     )
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
+    remote_rate_limit = storage_config.get('remote_rate_limit', None)
+    remote_rate_limit_flags = ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()
     umask = storage_config.get('umask', None)
     umask_flags = ('--umask', str(umask)) if umask else ()
     one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
@@ -101,7 +107,9 @@ def create_archive(
             repository=repository,
             archive_name_format=archive_name_format,
         ),
-    ) + sources + exclude_flags + compression_flags + one_file_system_flags + files_cache_flags + \
-        remote_path_flags + umask_flags + verbosity_flags
+    ) + sources + exclude_flags + compression_flags + remote_rate_limit_flags + \
+        one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \
+        verbosity_flags
 
+    logger.debug(' '.join(full_command))
     subprocess.check_call(full_command)

borgmatic/borg/extract.py

@@ -1,9 +1,13 @@
+import logging
 import sys
 import subprocess
 
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
+logger = logging.getLogger(__name__)
+
+
 def extract_last_archive_dry_run(verbosity, repository, remote_path=None):
     '''
     Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
@@ -37,4 +41,5 @@ def extract_last_archive_dry_run(verbosity, repository, remote_path=None):
         ),
     ) + remote_path_flags + verbosity_flags + list_flag
 
+    logger.debug(' '.join(full_extract_command))
     subprocess.check_call(full_extract_command)

borgmatic/borg/prune.py

@@ -1,8 +1,12 @@
+import logging
 import subprocess
 
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
+logger = logging.getLogger(__name__)
+
+
 def _make_prune_flags(retention_config):
     '''
     Given a retention config dict mapping from option name to value, tranform it into an iterable of
@@ -48,4 +52,5 @@ def prune_archives(verbosity, repository, retention_config, remote_path=None):
         for element in pair
     ) + remote_path_flags + verbosity_flags
 
+    logger.debug(' '.join(full_command))
     subprocess.check_call(full_command)

borgmatic/config/schema.yaml

@@ -24,9 +24,10 @@ map:
                 example: true
             files_cache:
                 type: scalar
-                desc: Mode in which to operate the files cache. See
-                      https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
-                      details.
+                desc: |
+                    Mode in which to operate the files cache. See
+                    https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
+                    details.
                 example: ctime,size,inode
             remote_path:
                 type: scalar
@@ -91,6 +92,10 @@ map:
                     https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
                     Defaults to no compression.
                 example: lz4
+            remote_rate_limit:
+                type: int
+                desc: Remote network upload rate limit in kiBytes/second.
+                example: 100
             umask:
                 type: scalar
                 desc: Umask to be used for borg create.

borgmatic/tests/unit/borg/test_create.py

@@ -231,6 +231,24 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
     )
 
 
+def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100'))
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'remote_rate_limit': 100},
+    )
+
+
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)