Replace "sequence" with "list" in docs for consistency.

Merge pull request #68 from diivi/feat/add-log-filename-to-hook-context

.drone.yml

@@ -24,6 +24,8 @@ clone:
 steps:
 - name: build
   image: alpine:3.13
+  environment:
+    TEST_CONTAINER: true
   pull: always
   commands:
     - scripts/run-full-tests

.eleventy.js

@@ -1,4 +1,5 @@
 const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
+const codeClipboard = require("eleventy-plugin-code-clipboard");
 const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language");
 const navigationPlugin = require("@11ty/eleventy-navigation");
 
@@ -6,6 +7,7 @@ module.exports = function(eleventyConfig) {
     eleventyConfig.addPlugin(pluginSyntaxHighlight);
     eleventyConfig.addPlugin(inclusiveLangPlugin);
     eleventyConfig.addPlugin(navigationPlugin);
+    eleventyConfig.addPlugin(codeClipboard);
 
     let markdownIt = require("markdown-it");
     let markdownItAnchor = require("markdown-it-anchor");
@@ -31,6 +33,7 @@ module.exports = function(eleventyConfig) {
         markdownIt(markdownItOptions)
             .use(markdownItAnchor, markdownItAnchorOptions)
             .use(markdownItReplaceLink)
+            .use(codeClipboard.markdownItCopyButton)
     );
 
     eleventyConfig.addPassthroughCopy({"docs/static": "static"});

.flake8

@@ -0,0 +1 @@
+select = Q0

NEWS

@@ -1,4 +1,76 @@
-1.7.8.dev0
+1.7.12.dev0
+ * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file.
+   See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+ * #666, #670: Fix error when running the "info" action with the "--match-archives" or "--archive"
+   flags. Also fix the "--match-archives"/"--archive" flags to correctly override the
+   "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions.
+ * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix"
+   options set.
+ * #672: Selectively shallow merge certain mappings or sequences when including configuration files.
+   See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge
+ * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag.
+   See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes
+ * Add optional support for running end-to-end tests and building documentation with rootless Podman
+   instead of Docker.
+
+1.7.11
+ * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives
+   get used for borgmatic actions that operate on multiple archives. Override this behavior with the
+   new "match_archives" option in the storage section. This change is "breaking" in that it silently
+   changes which archives get considered for "rlist", "prune", "check", etc. See the documentation
+   for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming
+ * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format"
+   auto-matching behavior and the "match_archives" option.
+ * #658: Add "--log-file-format" flag for customizing the log message format. See the documentation
+   for more information:
+   https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#logging-to-file
+ * #662: Fix regression in which the "check_repositories" option failed to match repositories.
+ * #663: Fix regression in which the "transfer" action produced a traceback.
+ * Add spellchecking of source code during test runs.
+
+1.7.10
+ * #396: When a database command errors, display and log the error message instead of swallowing it.
+ * #501: Optionally error if a source directory does not exist via "source_directories_must_exist"
+   option in borgmatic's location configuration.
+ * #576: Add support for "file://" paths within "repositories" option.
+ * #612: Define and use custom constants in borgmatic configuration files. See the documentation for
+   more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#constant-interpolation
+ * #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option
+   in borgmatic's storage configuration.
+ * #623: Fix confusing message when an error occurs running actions for a configuration file.
+ * #635: Add optional repository labels so you can select a repository via "--repository yourlabel"
+   at the command-line. See the configuration reference for more information:
+   https://torsion.org/borgmatic/docs/reference/configuration/
+ * #649: Add documentation on backing up a database running in a container:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+ * #655: Fix error when databases are configured and a source directory doesn't exist.
+ * Add code style plugins to enforce use of Python f-strings and prevent single-letter variables.
+   To join in the pedantry, refresh your test environment with "tox --recreate".
+ * Rename scripts/run-full-dev-tests to scripts/run-end-to-end-dev-tests and make it run end-to-end
+   tests only. Continue using tox to run unit and integration tests.
+
+1.7.9
+ * #295: Add a SQLite database dump/restore hook.
+ * #304: Change the default action order when no actions are specified on the command-line to:
+   "create", "prune", "compact", "check". If you'd like to retain the old ordering ("prune" and
+   "compact" first), then specify actions explicitly on the command-line.
+ * #304: Run any command-line actions in the order specified instead of using a fixed ordering.
+ * #564: Add "--repository" flag to all actions where it makes sense, so you can run borgmatic on
+   a single configured repository instead of all of them.
+ * #628: Add a Healthchecks "log" state to send borgmatic logs to Healthchecks without signalling
+   success or failure.
+ * #647: Add "--strip-components all" feature on the "extract" action to remove leading path
+   components of files you extract. Must be used with the "--path" flag.
+ * Add support for Python 3.11.
+
+1.7.8
+ * #620: With the "create" action and the "--list" ("--files") flag, only show excluded files at
+   verbosity 2.
  * #621: Add optional authentication to the ntfy monitoring hook.
  * With the "create" action, only one of "--list" ("--files") and "--progress" flags can be used.
    This lines up with the new behavior in Borg 2.0.0b5.
@@ -350,7 +422,7 @@
    configuration schema descriptions.
 
 1.5.6
- * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the
+ * #292: Allow before_backup and similar hooks to exit with a soft failure without altering the
    monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring
    services with a "start" status until after before_* hooks finish. Failures in before_* hooks
    still trigger a monitoring "fail" status.
@@ -419,7 +491,7 @@
  * For "list" and "info" actions, show repository names even at verbosity 0.
 
 1.4.22
- * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON ouput.
+ * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON output.
  * After a backup of a database dump in directory format, properly remove the dump directory.
  * In "borgmatic --help", don't expand $HOME in listing of default "--config" paths.
 
@@ -791,7 +863,7 @@
  * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files,
    editor swap files, etc.
  * #81: Document user-defined hooks run before/after backup, or on error.
- * Add code style guidelines to the documention.
+ * Add code style guidelines to the documentation.
 
 1.2.0
  * #61: Support for Borg --list option via borgmatic command-line to list all archives.

README.md

@@ -24,9 +24,10 @@ location:
 
     # Paths of local or remote repositories to backup to.
     repositories:
-        - ssh://1234@usw-s001.rsync.net/./backups.borg
-        - ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo
-        - /var/lib/backups/local.borg
+        - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo
+          label: borgbase
+        - path: /var/lib/backups/local.borg
+          label: local
 
 retention:
     # Retention policy for how many backups to keep.
@@ -67,6 +68,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 <a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
@@ -80,8 +82,8 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 Your first step is to [install and configure
 borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/).
 
-For additional documentation, check out the links above for <a
-href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
+For additional documentation, check out the links above (left panel on wide screens)
+for <a href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
 reference guides</a>.
 
 

borgmatic/actions/borg.py

@@ -16,9 +16,9 @@ def run_borg(
     if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, borg_arguments.repository
     ):
-        logger.info('{}: Running arbitrary Borg command'.format(repository))
+        logger.info(f'{repository["path"]}: Running arbitrary Borg command')
         archive_name = borgmatic.borg.rlist.resolve_archive_name(
-            repository,
+            repository['path'],
             borg_arguments.archive,
             storage,
             local_borg_version,
@@ -26,7 +26,7 @@ def run_borg(
             remote_path,
         )
         borgmatic.borg.borg.run_arbitrary_borg(
-            repository,
+            repository['path'],
             storage,
             local_borg_version,
             options=borg_arguments.options,

borgmatic/actions/break_lock.py

@@ -15,7 +15,11 @@ def run_break_lock(
     if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, break_lock_arguments.repository
     ):
-        logger.info(f'{repository}: Breaking repository and cache locks')
+        logger.info(f'{repository["path"]}: Breaking repository and cache locks')
         borgmatic.borg.break_lock.break_lock(
-            repository, storage, local_borg_version, local_path=local_path, remote_path=remote_path,
+            repository['path'],
+            storage,
+            local_borg_version,
+            local_path=local_path,
+            remote_path=remote_path,
         )

borgmatic/actions/check.py

@@ -1,6 +1,7 @@
 import logging
 
 import borgmatic.borg.check
+import borgmatic.config.validate
 import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
@@ -23,6 +24,11 @@ def run_check(
     '''
     Run the "check" action for the given repository.
     '''
+    if check_arguments.repository and not borgmatic.config.validate.repositories_match(
+        repository, check_arguments.repository
+    ):
+        return
+
     borgmatic.hooks.command.execute_hook(
         hooks.get('before_check'),
         hooks.get('umask'),
@@ -31,9 +37,9 @@ def run_check(
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info('{}: Running consistency checks'.format(repository))
+    logger.info(f'{repository["path"]}: Running consistency checks')
     borgmatic.borg.check.check_archives(
-        repository,
+        repository['path'],
         location,
         storage,
         consistency,

borgmatic/actions/compact.py

@@ -2,6 +2,7 @@ import logging
 
 import borgmatic.borg.compact
 import borgmatic.borg.feature
+import borgmatic.config.validate
 import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
@@ -24,6 +25,11 @@ def run_compact(
     '''
     Run the "compact" action for the given repository.
     '''
+    if compact_arguments.repository and not borgmatic.config.validate.repositories_match(
+        repository, compact_arguments.repository
+    ):
+        return
+
     borgmatic.hooks.command.execute_hook(
         hooks.get('before_compact'),
         hooks.get('umask'),
@@ -33,10 +39,10 @@ def run_compact(
         **hook_context,
     )
     if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
-        logger.info('{}: Compacting segments{}'.format(repository, dry_run_label))
+        logger.info(f'{repository["path"]}: Compacting segments{dry_run_label}')
         borgmatic.borg.compact.compact_segments(
             global_arguments.dry_run,
-            repository,
+            repository['path'],
             storage,
             local_borg_version,
             local_path=local_path,
@@ -46,7 +52,7 @@ def run_compact(
             threshold=compact_arguments.threshold,
         )
     else:  # pragma: nocover
-        logger.info('{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository))
+        logger.info(f'{repository["path"]}: Skipping compact (only available/needed in Borg 1.2+)')
     borgmatic.hooks.command.execute_hook(
         hooks.get('after_compact'),
         hooks.get('umask'),

borgmatic/actions/create.py

@@ -2,6 +2,7 @@ import json
 import logging
 
 import borgmatic.borg.create
+import borgmatic.config.validate
 import borgmatic.hooks.command
 import borgmatic.hooks.dispatch
 import borgmatic.hooks.dump
@@ -28,6 +29,11 @@ def run_create(
 
     If create_arguments.json is True, yield the JSON output from creating the archive.
     '''
+    if create_arguments.repository and not borgmatic.config.validate.repositories_match(
+        repository, create_arguments.repository
+    ):
+        return
+
     borgmatic.hooks.command.execute_hook(
         hooks.get('before_backup'),
         hooks.get('umask'),
@@ -36,11 +42,11 @@ def run_create(
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
+    logger.info(f'{repository["path"]}: Creating archive{dry_run_label}')
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
         'remove_database_dumps',
         hooks,
-        repository,
+        repository['path'],
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         location,
         global_arguments.dry_run,
@@ -48,7 +54,7 @@ def run_create(
     active_dumps = borgmatic.hooks.dispatch.call_hooks(
         'dump_databases',
         hooks,
-        repository,
+        repository['path'],
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         location,
         global_arguments.dry_run,
@@ -57,7 +63,7 @@ def run_create(
 
     json_output = borgmatic.borg.create.create_archive(
         global_arguments.dry_run,
-        repository,
+        repository['path'],
         location,
         storage,
         local_borg_version,

borgmatic/actions/export_tar.py

@@ -23,13 +23,13 @@ def run_export_tar(
         repository, export_tar_arguments.repository
     ):
         logger.info(
-            '{}: Exporting archive {} as tar file'.format(repository, export_tar_arguments.archive)
+            f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file'
         )
         borgmatic.borg.export_tar.export_tar_archive(
             global_arguments.dry_run,
-            repository,
+            repository['path'],
             borgmatic.borg.rlist.resolve_archive_name(
-                repository,
+                repository['path'],
                 export_tar_arguments.archive,
                 storage,
                 local_borg_version,

borgmatic/actions/extract.py

@@ -35,12 +35,12 @@ def run_extract(
     if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, extract_arguments.repository
     ):
-        logger.info('{}: Extracting archive {}'.format(repository, extract_arguments.archive))
+        logger.info(f'{repository["path"]}: Extracting archive {extract_arguments.archive}')
         borgmatic.borg.extract.extract_archive(
             global_arguments.dry_run,
-            repository,
+            repository['path'],
             borgmatic.borg.rlist.resolve_archive_name(
-                repository,
+                repository['path'],
                 extract_arguments.archive,
                 storage,
                 local_borg_version,

borgmatic/actions/info.py

@@ -20,9 +20,9 @@ def run_info(
         repository, info_arguments.repository
     ):
         if not info_arguments.json:  # pragma: nocover
-            logger.answer(f'{repository}: Displaying archive summary information')
+            logger.answer(f'{repository["path"]}: Displaying archive summary information')
         info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name(
-            repository,
+            repository['path'],
             info_arguments.archive,
             storage,
             local_borg_version,
@@ -30,7 +30,7 @@ def run_info(
             remote_path,
         )
         json_output = borgmatic.borg.info.display_archives_info(
-            repository,
+            repository['path'],
             storage,
             local_borg_version,
             info_arguments=info_arguments,

borgmatic/actions/list.py

@@ -20,11 +20,11 @@ def run_list(
     ):
         if not list_arguments.json:  # pragma: nocover
             if list_arguments.find_paths:
-                logger.answer(f'{repository}: Searching archives')
+                logger.answer(f'{repository["path"]}: Searching archives')
             elif not list_arguments.archive:
-                logger.answer(f'{repository}: Listing archives')
+                logger.answer(f'{repository["path"]}: Listing archives')
         list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name(
-            repository,
+            repository['path'],
             list_arguments.archive,
             storage,
             local_borg_version,
@@ -32,7 +32,7 @@ def run_list(
             remote_path,
         )
         json_output = borgmatic.borg.list.list_archive(
-            repository,
+            repository['path'],
             storage,
             local_borg_version,
             list_arguments=list_arguments,

borgmatic/actions/mount.py

@@ -17,14 +17,14 @@ def run_mount(
         repository, mount_arguments.repository
     ):
         if mount_arguments.archive:
-            logger.info('{}: Mounting archive {}'.format(repository, mount_arguments.archive))
+            logger.info(f'{repository["path"]}: Mounting archive {mount_arguments.archive}')
         else:  # pragma: nocover
-            logger.info('{}: Mounting repository'.format(repository))
+            logger.info(f'{repository["path"]}: Mounting repository')
 
         borgmatic.borg.mount.mount_archive(
-            repository,
+            repository['path'],
             borgmatic.borg.rlist.resolve_archive_name(
-                repository,
+                repository['path'],
                 mount_arguments.archive,
                 storage,
                 local_borg_version,

borgmatic/actions/prune.py

@@ -1,6 +1,7 @@
 import logging
 
 import borgmatic.borg.prune
+import borgmatic.config.validate
 import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
@@ -23,6 +24,11 @@ def run_prune(
     '''
     Run the "prune" action for the given repository.
     '''
+    if prune_arguments.repository and not borgmatic.config.validate.repositories_match(
+        repository, prune_arguments.repository
+    ):
+        return
+
     borgmatic.hooks.command.execute_hook(
         hooks.get('before_prune'),
         hooks.get('umask'),
@@ -31,10 +37,10 @@ def run_prune(
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
+    logger.info(f'{repository["path"]}: Pruning archives{dry_run_label}')
     borgmatic.borg.prune.prune_archives(
         global_arguments.dry_run,
-        repository,
+        repository['path'],
         storage,
         retention,
         local_borg_version,

borgmatic/actions/rcreate.py

@@ -23,10 +23,10 @@ def run_rcreate(
     ):
         return
 
-    logger.info('{}: Creating repository'.format(repository))
+    logger.info(f'{repository["path"]}: Creating repository')
     borgmatic.borg.rcreate.create_repository(
         global_arguments.dry_run,
-        repository,
+        repository['path'],
         storage,
         local_borg_version,
         rcreate_arguments.encryption_mode,

borgmatic/actions/restore.py

@@ -256,22 +256,34 @@ def run_restore(
         return
 
     logger.info(
-        '{}: Restoring databases from archive {}'.format(repository, restore_arguments.archive)
+        f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}'
     )
+
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
         'remove_database_dumps',
         hooks,
-        repository,
+        repository['path'],
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         location,
         global_arguments.dry_run,
     )
 
     archive_name = borgmatic.borg.rlist.resolve_archive_name(
-        repository, restore_arguments.archive, storage, local_borg_version, local_path, remote_path,
+        repository['path'],
+        restore_arguments.archive,
+        storage,
+        local_borg_version,
+        local_path,
+        remote_path,
     )
     archive_database_names = collect_archive_database_names(
-        repository, archive_name, location, storage, local_borg_version, local_path, remote_path,
+        repository['path'],
+        archive_name,
+        location,
+        storage,
+        local_borg_version,
+        local_path,
+        remote_path,
     )
     restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
     found_names = set()
@@ -291,7 +303,7 @@ def run_restore(
 
             found_names.add(database_name)
             restore_single_database(
-                repository,
+                repository['path'],
                 location,
                 storage,
                 hooks,
@@ -320,7 +332,7 @@ def run_restore(
             database['name'] = database_name
 
             restore_single_database(
-                repository,
+                repository['path'],
                 location,
                 storage,
                 hooks,
@@ -336,7 +348,7 @@ def run_restore(
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
         'remove_database_dumps',
         hooks,
-        repository,
+        repository['path'],
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         location,
         global_arguments.dry_run,

borgmatic/actions/rinfo.py

@@ -19,9 +19,10 @@ def run_rinfo(
         repository, rinfo_arguments.repository
     ):
         if not rinfo_arguments.json:  # pragma: nocover
-            logger.answer('{}: Displaying repository summary information'.format(repository))
+            logger.answer(f'{repository["path"]}: Displaying repository summary information')
+
         json_output = borgmatic.borg.rinfo.display_repository_info(
-            repository,
+            repository['path'],
             storage,
             local_borg_version,
             rinfo_arguments=rinfo_arguments,

borgmatic/actions/rlist.py

@@ -19,9 +19,10 @@ def run_rlist(
         repository, rlist_arguments.repository
     ):
         if not rlist_arguments.json:  # pragma: nocover
-            logger.answer('{}: Listing repository'.format(repository))
+            logger.answer(f'{repository["path"]}: Listing repository')
+
         json_output = borgmatic.borg.rlist.list_repository(
-            repository,
+            repository['path'],
             storage,
             local_borg_version,
             rlist_arguments=rlist_arguments,

borgmatic/actions/transfer.py

@@ -17,10 +17,10 @@ def run_transfer(
     '''
     Run the "transfer" action for the given repository.
     '''
-    logger.info(f'{repository}: Transferring archives to repository')
+    logger.info(f'{repository["path"]}: Transferring archives to repository')
     borgmatic.borg.transfer.transfer_archives(
         global_arguments.dry_run,
-        repository,
+        repository['path'],
         storage,
         local_borg_version,
         transfer_arguments,

borgmatic/borg/borg.py

@@ -13,7 +13,7 @@ BORG_SUBCOMMANDS_WITHOUT_REPOSITORY = (('debug', 'info'), ('debug', 'convert-pro
 
 
 def run_arbitrary_borg(
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     options,
@@ -44,10 +44,10 @@ def run_arbitrary_borg(
         repository_archive_flags = ()
     elif archive:
         repository_archive_flags = flags.make_repository_archive_flags(
-            repository, archive, local_borg_version
+            repository_path, archive, local_borg_version
         )
     else:
-        repository_archive_flags = flags.make_repository_flags(repository, local_borg_version)
+        repository_archive_flags = flags.make_repository_flags(repository_path, local_borg_version)
 
     full_command = (
         (local_path,)

borgmatic/borg/break_lock.py

@@ -7,7 +7,7 @@ logger = logging.getLogger(__name__)
 
 
 def break_lock(
-    repository, storage_config, local_borg_version, local_path='borg', remote_path=None,
+    repository_path, storage_config, local_borg_version, local_path='borg', remote_path=None,
 ):
     '''
     Given a local or remote repository path, a storage configuration dict, the local Borg version,
@@ -24,7 +24,7 @@ def break_lock(
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + flags.make_repository_flags(repository, local_borg_version)
+        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     borg_environment = environment.make_environment(storage_config)

borgmatic/borg/check.py

@@ -12,7 +12,6 @@ DEFAULT_CHECKS = (
     {'name': 'repository', 'frequency': '1 month'},
     {'name': 'archives', 'frequency': '1 month'},
 )
-DEFAULT_PREFIX = '{hostname}-'
 
 
 logger = logging.getLogger(__name__)
@@ -139,16 +138,17 @@ def filter_checks_on_frequency(
         if datetime.datetime.now() < check_time + frequency_delta:
             remaining = check_time + frequency_delta - datetime.datetime.now()
             logger.info(
-                f"Skipping {check} check due to configured frequency; {remaining} until next check"
+                f'Skipping {check} check due to configured frequency; {remaining} until next check'
             )
             filtered_checks.remove(check)
 
     return tuple(filtered_checks)
 
 
-def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
+def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None):
     '''
-    Given the local Borg version and a parsed sequence of checks, transform the checks into tuple of
+    Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the
+    check last value, and a consistency check prefix, transform the checks into tuple of
     command-line flags.
 
     For example, given parsed checks of:
@@ -174,10 +174,21 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
 
     if 'archives' in checks:
         last_flags = ('--last', str(check_last)) if check_last else ()
-        if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
-            match_archives_flags = ('--match-archives', f'sh:{prefix}*') if prefix else ()
-        else:
-            match_archives_flags = ('--glob-archives', f'{prefix}*') if prefix else ()
+        match_archives_flags = (
+            (
+                ('--match-archives', f'sh:{prefix}*')
+                if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
+                else ('--glob-archives', f'{prefix}*')
+            )
+            if prefix
+            else (
+                flags.make_match_archives_flags(
+                    storage_config.get('match_archives'),
+                    storage_config.get('archive_name_format'),
+                    local_borg_version,
+                )
+            )
+        )
     else:
         last_flags = ()
         match_archives_flags = ()
@@ -196,7 +207,7 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
         return common_flags
 
     return (
-        tuple('--{}-only'.format(check) for check in checks if check in ('repository', 'archives'))
+        tuple(f'--{check}-only' for check in checks if check in ('repository', 'archives'))
         + common_flags
     )
 
@@ -243,7 +254,7 @@ def read_check_time(path):
 
 
 def check_archives(
-    repository,
+    repository_path,
     location_config,
     storage_config,
     consistency_config,
@@ -268,7 +279,7 @@ def check_archives(
     try:
         borg_repository_id = json.loads(
             rinfo.display_repository_info(
-                repository,
+                repository_path,
                 storage_config,
                 local_borg_version,
                 argparse.Namespace(json=True),
@@ -277,7 +288,7 @@ def check_archives(
             )
         )['repository']['id']
     except (json.JSONDecodeError, KeyError):
-        raise ValueError(f'Cannot determine Borg repository ID for {repository}')
+        raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
 
     checks = filter_checks_on_frequency(
         location_config,
@@ -291,7 +302,7 @@ def check_archives(
     extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
 
     if set(checks).intersection({'repository', 'archives', 'data'}):
-        lock_wait = storage_config.get('lock_wait', None)
+        lock_wait = storage_config.get('lock_wait')
 
         verbosity_flags = ()
         if logger.isEnabledFor(logging.INFO):
@@ -299,18 +310,18 @@ def check_archives(
         if logger.isEnabledFor(logging.DEBUG):
             verbosity_flags = ('--debug', '--show-rc')
 
-        prefix = consistency_config.get('prefix', DEFAULT_PREFIX)
+        prefix = consistency_config.get('prefix')
 
         full_command = (
             (local_path, 'check')
             + (('--repair',) if repair else ())
-            + make_check_flags(local_borg_version, checks, check_last, prefix)
+            + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix)
             + (('--remote-path', remote_path) if remote_path else ())
             + (('--lock-wait', str(lock_wait)) if lock_wait else ())
             + verbosity_flags
             + (('--progress',) if progress else ())
             + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-            + flags.make_repository_flags(repository, local_borg_version)
+            + flags.make_repository_flags(repository_path, local_borg_version)
         )
 
         borg_environment = environment.make_environment(storage_config)
@@ -329,6 +340,6 @@ def check_archives(
 
     if 'extract' in checks:
         extract.extract_last_archive_dry_run(
-            storage_config, local_borg_version, repository, lock_wait, local_path, remote_path
+            storage_config, local_borg_version, repository_path, lock_wait, local_path, remote_path
         )
         write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract'))

borgmatic/borg/compact.py

@@ -8,7 +8,7 @@ logger = logging.getLogger(__name__)
 
 def compact_segments(
     dry_run,
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     local_path='borg',
@@ -36,11 +36,11 @@ def compact_segments(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_flags(repository, local_borg_version)
+        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     if dry_run:
-        logging.info(f'{repository}: Skipping compact (dry run)')
+        logging.info(f'{repository_path}: Skipping compact (dry run)')
         return
 
     execute_command(

borgmatic/borg/create.py

@@ -196,7 +196,28 @@ def make_exclude_flags(location_config, exclude_filename=None):
     )
 
 
-DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+def make_list_filter_flags(local_borg_version, dry_run):
+    '''
+    Given the local Borg version and whether this is a dry run, return the corresponding flags for
+    passing to "--list --filter". The general idea is that excludes are shown for a dry run or when
+    the verbosity is debug.
+    '''
+    base_flags = 'AME'
+    show_excludes = logger.isEnabledFor(logging.DEBUG)
+
+    if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version):
+        if show_excludes or dry_run:
+            return f'{base_flags}+-'
+        else:
+            return base_flags
+
+    if show_excludes:
+        return f'{base_flags}x-'
+    else:
+        return f'{base_flags}-'
+
+
+DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
 
 
 def collect_borgmatic_source_directories(borgmatic_source_directory):
@@ -285,9 +306,23 @@ def collect_special_file_paths(
     )
 
 
+def check_all_source_directories_exist(source_directories):
+    '''
+    Given a sequence of source directories, check that they all exist. If any do not, raise an
+    exception.
+    '''
+    missing_directories = [
+        source_directory
+        for source_directory in source_directories
+        if not os.path.exists(source_directory)
+    ]
+    if missing_directories:
+        raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
+
+
 def create_archive(
     dry_run,
-    repository,
+    repository_path,
     location_config,
     storage_config,
     local_borg_version,
@@ -310,6 +345,8 @@ def create_archive(
     borgmatic_source_directories = expand_directories(
         collect_borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
     )
+    if location_config.get('source_directories_must_exist', False):
+        check_all_source_directories_exist(location_config.get('source_directories'))
     sources = deduplicate_directories(
         map_directories_to_devices(
             expand_directories(
@@ -343,6 +380,7 @@ def create_archive(
     upload_rate_limit = storage_config.get('upload_rate_limit', None)
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
+    list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
     files_cache = location_config.get('files_cache')
     archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
     extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
@@ -373,7 +411,7 @@ def create_archive(
 
     if stream_processes and location_config.get('read_special') is False:
         logger.warning(
-            f'{repository}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
+            f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
         )
 
     create_command = (
@@ -401,10 +439,16 @@ def create_archive(
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--list', '--filter', 'AMEx+-') if list_files and not json and not progress else ())
+        + (
+            ('--list', '--filter', list_filter_flags)
+            if list_files and not json and not progress
+            else ()
+        )
         + (('--dry-run',) if dry_run else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_archive_flags(repository, archive_name_format, local_borg_version)
+        + flags.make_repository_archive_flags(
+            repository_path, archive_name_format, local_borg_version
+        )
         + (sources if not pattern_file else ())
     )
 
@@ -424,7 +468,7 @@ def create_archive(
     # If database hooks are enabled (as indicated by streaming processes), exclude files that might
     # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
     if stream_processes and not location_config.get('read_special'):
-        logger.debug(f'{repository}: Collecting special file paths')
+        logger.debug(f'{repository_path}: Collecting special file paths')
         special_file_paths = collect_special_file_paths(
             create_command,
             local_path,
@@ -435,7 +479,7 @@ def create_archive(
 
         if special_file_paths:
             logger.warning(
-                f'{repository}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}'
+                f'{repository_path}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}'
             )
             exclude_file = write_pattern_file(
                 expand_home_directories(

borgmatic/borg/environment.py

@@ -2,6 +2,7 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_base_directory': 'BORG_BASE_DIR',
     'borg_config_directory': 'BORG_CONFIG_DIR',
     'borg_cache_directory': 'BORG_CACHE_DIR',
+    'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
     'borg_security_directory': 'BORG_SECURITY_DIR',
     'borg_keys_directory': 'BORG_KEYS_DIR',
     'encryption_passcommand': 'BORG_PASSCOMMAND',
@@ -27,7 +28,7 @@ def make_environment(storage_config):
         value = storage_config.get(option_name)
 
         if value:
-            environment[environment_variable_name] = value
+            environment[environment_variable_name] = str(value)
 
     for (
         option_name,

borgmatic/borg/export_tar.py

@@ -1,5 +1,4 @@
 import logging
-import os
 
 import borgmatic.logger
 from borgmatic.borg import environment, flags
@@ -10,7 +9,7 @@ logger = logging.getLogger(__name__)
 
 def export_tar_archive(
     dry_run,
-    repository,
+    repository_path,
     archive,
     paths,
     destination_path,
@@ -46,11 +45,7 @@ def export_tar_archive(
         + (('--dry-run',) if dry_run else ())
         + (('--tar-filter', tar_filter) if tar_filter else ())
         + (('--strip-components', str(strip_components)) if strip_components else ())
-        + flags.make_repository_archive_flags(
-            repository if ':' in repository else os.path.abspath(repository),
-            archive,
-            local_borg_version,
-        )
+        + flags.make_repository_archive_flags(repository_path, archive, local_borg_version,)
         + (destination_path,)
         + (tuple(paths) if paths else ())
     )
@@ -61,7 +56,7 @@ def export_tar_archive(
         output_log_level = logging.INFO
 
     if dry_run:
-        logging.info('{}: Skipping export to tar file (dry run)'.format(repository))
+        logging.info(f'{repository_path}: Skipping export to tar file (dry run)')
         return
 
     execute_command(

borgmatic/borg/extract.py

@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
 def extract_last_archive_dry_run(
     storage_config,
     local_borg_version,
-    repository,
+    repository_path,
     lock_wait=None,
     local_path='borg',
     remote_path=None,
@@ -30,7 +30,7 @@ def extract_last_archive_dry_run(
 
     try:
         last_archive_name = rlist.resolve_archive_name(
-            repository, 'latest', storage_config, local_borg_version, local_path, remote_path
+            repository_path, 'latest', storage_config, local_borg_version, local_path, remote_path
         )
     except ValueError:
         logger.warning('No archives found. Skipping extract consistency check.')
@@ -44,7 +44,9 @@ def extract_last_archive_dry_run(
         + lock_wait_flags
         + verbosity_flags
         + list_flag
-        + flags.make_repository_archive_flags(repository, last_archive_name, local_borg_version)
+        + flags.make_repository_archive_flags(
+            repository_path, last_archive_name, local_borg_version
+        )
     )
 
     execute_command(
@@ -87,6 +89,13 @@ def extract_archive(
     else:
         numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_ids') else ()
 
+    if strip_components == 'all':
+        if not paths:
+            raise ValueError('The --strip-components flag with "all" requires at least one --path')
+
+        # Calculate the maximum number of leading path components of the given paths.
+        strip_components = max(0, *(len(path.split(os.path.sep)) - 1 for path in paths))
+
     full_command = (
         (local_path, 'extract')
         + (('--remote-path', remote_path) if remote_path else ())
@@ -99,11 +108,7 @@ def extract_archive(
         + (('--strip-components', str(strip_components)) if strip_components else ())
         + (('--progress',) if progress else ())
         + (('--stdout',) if extract_to_stdout else ())
-        + flags.make_repository_archive_flags(
-            repository if ':' in repository else os.path.abspath(repository),
-            archive,
-            local_borg_version,
-        )
+        + flags.make_repository_archive_flags(repository, archive, local_borg_version,)
         + (tuple(paths) if paths else ())
     )
 

borgmatic/borg/feature.py

@@ -14,6 +14,7 @@ class Feature(Enum):
     RLIST = 8
     RINFO = 9
     MATCH_ARCHIVES = 10
+    EXCLUDED_FILES_MINUS = 11
 
 
 FEATURE_TO_MINIMUM_BORG_VERSION = {
@@ -27,6 +28,7 @@ FEATURE_TO_MINIMUM_BORG_VERSION = {
     Feature.RLIST: parse_version('2.0.0a2'),  # borg rlist
     Feature.RINFO: parse_version('2.0.0a2'),  # borg rinfo
     Feature.MATCH_ARCHIVES: parse_version('2.0.0b3'),  # borg --match-archives
+    Feature.EXCLUDED_FILES_MINUS: parse_version('2.0.0b5'),  # --list --filter uses "-" for excludes
 }
 
 

borgmatic/borg/flags.py

@@ -1,4 +1,5 @@
 import itertools
+import re
 
 from borgmatic.borg import feature
 
@@ -10,7 +11,7 @@ def make_flags(name, value):
     if not value:
         return ()
 
-    flag = '--{}'.format(name.replace('_', '-'))
+    flag = f"--{name.replace('_', '-')}"
 
     if value is True:
         return (flag,)
@@ -33,7 +34,7 @@ def make_flags_from_arguments(arguments, excludes=()):
     )
 
 
-def make_repository_flags(repository, local_borg_version):
+def make_repository_flags(repository_path, local_borg_version):
     '''
     Given the path of a Borg repository and the local Borg version, return Borg-version-appropriate
     command-line flags (as a tuple) for selecting that repository.
@@ -42,17 +43,41 @@ def make_repository_flags(repository, local_borg_version):
         ('--repo',)
         if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version)
         else ()
-    ) + (repository,)
+    ) + (repository_path,)
 
 
-def make_repository_archive_flags(repository, archive, local_borg_version):
+def make_repository_archive_flags(repository_path, archive, local_borg_version):
     '''
     Given the path of a Borg repository, an archive name or pattern, and the local Borg version,
     return Borg-version-appropriate command-line flags (as a tuple) for selecting that repository
     and archive.
     '''
     return (
-        ('--repo', repository, archive)
+        ('--repo', repository_path, archive)
         if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version)
-        else (f'{repository}::{archive}',)
+        else (f'{repository_path}::{archive}',)
     )
+
+
+def make_match_archives_flags(match_archives, archive_name_format, local_borg_version):
+    '''
+    Return match archives flags based on the given match archives value, if any. If it isn't set,
+    return match archives flags to match archives created with the given archive name format, if
+    any. This is done by replacing certain archive name format placeholders for ephemeral data (like
+    "{now}") with globs.
+    '''
+    if match_archives:
+        if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
+            return ('--match-archives', match_archives)
+        else:
+            return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
+
+    if not archive_name_format:
+        return ()
+
+    derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format)
+
+    if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
+        return ('--match-archives', f'sh:{derived_match_archives}')
+    else:
+        return ('--glob-archives', f'{derived_match_archives}')

borgmatic/borg/info.py

@@ -8,7 +8,7 @@ logger = logging.getLogger(__name__)
 
 
 def display_archives_info(
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     info_arguments,
@@ -44,17 +44,20 @@ def display_archives_info(
                 else flags.make_flags('glob-archives', f'{info_arguments.prefix}*')
             )
             if info_arguments.prefix
-            else ()
+            else (
+                flags.make_match_archives_flags(
+                    info_arguments.match_archives
+                    or info_arguments.archive
+                    or storage_config.get('match_archives'),
+                    storage_config.get('archive_name_format'),
+                    local_borg_version,
+                )
+            )
         )
         + flags.make_flags_from_arguments(
-            info_arguments, excludes=('repository', 'archive', 'prefix')
-        )
-        + flags.make_repository_flags(repository, local_borg_version)
-        + (
-            flags.make_flags('match-archives', info_arguments.archive)
-            if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
-            else flags.make_flags('glob-archives', info_arguments.archive)
+            info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives')
         )
+        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     if info_arguments.json:

borgmatic/borg/list.py

@@ -21,7 +21,7 @@ MAKE_FLAGS_EXCLUDES = (
 
 
 def make_list_command(
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     list_arguments,
@@ -52,10 +52,10 @@ def make_list_command(
         + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + (
             flags.make_repository_archive_flags(
-                repository, list_arguments.archive, local_borg_version
+                repository_path, list_arguments.archive, local_borg_version
             )
             if list_arguments.archive
-            else flags.make_repository_flags(repository, local_borg_version)
+            else flags.make_repository_flags(repository_path, local_borg_version)
         )
         + (tuple(list_arguments.paths) if list_arguments.paths else ())
     )
@@ -86,7 +86,7 @@ def make_find_paths(find_paths):
 
 
 def capture_archive_listing(
-    repository,
+    repository_path,
     archive,
     storage_config,
     local_borg_version,
@@ -104,16 +104,16 @@ def capture_archive_listing(
     return tuple(
         execute_command_and_capture_output(
             make_list_command(
-                repository,
+                repository_path,
                 storage_config,
                 local_borg_version,
                 argparse.Namespace(
-                    repository=repository,
+                    repository=repository_path,
                     archive=archive,
                     paths=[f'sh:{list_path}'],
                     find_paths=None,
                     json=None,
-                    format='{path}{NL}',
+                    format='{path}{NL}',  # noqa: FS003
                 ),
                 local_path,
                 remote_path,
@@ -126,7 +126,7 @@ def capture_archive_listing(
 
 
 def list_archive(
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     list_arguments,
@@ -149,7 +149,7 @@ def list_archive(
             )
 
         rlist_arguments = argparse.Namespace(
-            repository=repository,
+            repository=repository_path,
             short=list_arguments.short,
             format=list_arguments.format,
             json=list_arguments.json,
@@ -160,7 +160,12 @@ def list_archive(
             last=list_arguments.last,
         )
         return rlist.list_repository(
-            repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path
+            repository_path,
+            storage_config,
+            local_borg_version,
+            rlist_arguments,
+            local_path,
+            remote_path,
         )
 
     if list_arguments.archive:
@@ -181,7 +186,7 @@ def list_archive(
     # getting a list of archives to search.
     if list_arguments.find_paths and not list_arguments.archive:
         rlist_arguments = argparse.Namespace(
-            repository=repository,
+            repository=repository_path,
             short=True,
             format=None,
             json=None,
@@ -196,7 +201,7 @@ def list_archive(
         archive_lines = tuple(
             execute_command_and_capture_output(
                 rlist.make_rlist_command(
-                    repository,
+                    repository_path,
                     storage_config,
                     local_borg_version,
                     rlist_arguments,
@@ -213,7 +218,7 @@ def list_archive(
 
     # For each archive listed by Borg, run list on the contents of that archive.
     for archive in archive_lines:
-        logger.answer(f'{repository}: Listing archive {archive}')
+        logger.answer(f'{repository_path}: Listing archive {archive}')
 
         archive_arguments = copy.copy(list_arguments)
         archive_arguments.archive = archive
@@ -224,7 +229,7 @@ def list_archive(
             setattr(archive_arguments, name, None)
 
         main_command = make_list_command(
-            repository,
+            repository_path,
             storage_config,
             local_borg_version,
             archive_arguments,

borgmatic/borg/mount.py

@@ -7,7 +7,7 @@ logger = logging.getLogger(__name__)
 
 
 def mount_archive(
-    repository,
+    repository_path,
     archive,
     mount_point,
     paths,
@@ -38,7 +38,7 @@ def mount_archive(
         + (('-o', options) if options else ())
         + (
             (
-                flags.make_repository_flags(repository, local_borg_version)
+                flags.make_repository_flags(repository_path, local_borg_version)
                 + (
                     ('--match-archives', archive)
                     if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
@@ -47,9 +47,9 @@ def mount_archive(
             )
             if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version)
             else (
-                flags.make_repository_archive_flags(repository, archive, local_borg_version)
+                flags.make_repository_archive_flags(repository_path, archive, local_borg_version)
                 if archive
-                else flags.make_repository_flags(repository, local_borg_version)
+                else flags.make_repository_flags(repository_path, local_borg_version)
             )
         )
         + (mount_point,)

borgmatic/borg/prune.py

@@ -7,10 +7,10 @@ from borgmatic.execute import execute_command
 logger = logging.getLogger(__name__)
 
 
-def make_prune_flags(retention_config, local_borg_version):
+def make_prune_flags(storage_config, retention_config, local_borg_version):
     '''
-    Given a retention config dict mapping from option name to value, tranform it into an iterable of
-    command-line name-value flag pairs.
+    Given a retention config dict mapping from option name to value, transform it into an sequence of
+    command-line flags.
 
     For example, given a retention config of:
 
@@ -24,22 +24,32 @@ def make_prune_flags(retention_config, local_borg_version):
         )
     '''
     config = retention_config.copy()
-    prefix = config.pop('prefix', '{hostname}-')
+    prefix = config.pop('prefix', None)
 
-    if prefix:
-        if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
-            config['match_archives'] = f'sh:{prefix}*'
-        else:
-            config['glob_archives'] = f'{prefix}*'
-
-    return (
+    flag_pairs = (
         ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
     )
 
+    return tuple(element for pair in flag_pairs for element in pair) + (
+        (
+            ('--match-archives', f'sh:{prefix}*')
+            if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
+            else ('--glob-archives', f'{prefix}*')
+        )
+        if prefix
+        else (
+            flags.make_match_archives_flags(
+                storage_config.get('match_archives'),
+                storage_config.get('archive_name_format'),
+                local_borg_version,
+            )
+        )
+    )
+
 
 def prune_archives(
     dry_run,
-    repository,
+    repository_path,
     storage_config,
     retention_config,
     local_borg_version,
@@ -60,11 +70,7 @@ def prune_archives(
 
     full_command = (
         (local_path, 'prune')
-        + tuple(
-            element
-            for pair in make_prune_flags(retention_config, local_borg_version)
-            for element in pair
-        )
+        + make_prune_flags(storage_config, retention_config, local_borg_version)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
@@ -74,7 +80,7 @@ def prune_archives(
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_flags(repository, local_borg_version)
+        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     if stats or list_archives:

borgmatic/borg/rcreate.py

@@ -13,7 +13,7 @@ RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
 
 def create_repository(
     dry_run,
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     encryption_mode,
@@ -33,14 +33,14 @@ def create_repository(
     '''
     try:
         rinfo.display_repository_info(
-            repository,
+            repository_path,
             storage_config,
             local_borg_version,
             argparse.Namespace(json=True),
             local_path,
             remote_path,
         )
-        logger.info(f'{repository}: Repository already exists. Skipping creation.')
+        logger.info(f'{repository_path}: Repository already exists. Skipping creation.')
         return
     except subprocess.CalledProcessError as error:
         if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
@@ -65,11 +65,11 @@ def create_repository(
         + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_flags(repository, local_borg_version)
+        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     if dry_run:
-        logging.info(f'{repository}: Skipping repository creation (dry run)')
+        logging.info(f'{repository_path}: Skipping repository creation (dry run)')
         return
 
     # Do not capture output here, so as to support interactive prompts.

borgmatic/borg/rinfo.py

@@ -8,7 +8,7 @@ logger = logging.getLogger(__name__)
 
 
 def display_repository_info(
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     rinfo_arguments,
@@ -43,7 +43,7 @@ def display_repository_info(
         + flags.make_flags('remote-path', remote_path)
         + flags.make_flags('lock-wait', lock_wait)
         + (('--json',) if rinfo_arguments.json else ())
-        + flags.make_repository_flags(repository, local_borg_version)
+        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     extra_environment = environment.make_environment(storage_config)

borgmatic/borg/rlist.py

@@ -8,7 +8,12 @@ logger = logging.getLogger(__name__)
 
 
 def resolve_archive_name(
-    repository, archive, storage_config, local_borg_version, local_path='borg', remote_path=None
+    repository_path,
+    archive,
+    storage_config,
+    local_borg_version,
+    local_path='borg',
+    remote_path=None,
 ):
     '''
     Given a local or remote repository path, an archive name, a storage config dict, a local Borg
@@ -17,7 +22,7 @@ def resolve_archive_name(
 
     Raise ValueError if "latest" is given but there are no archives in the repository.
     '''
-    if archive != "latest":
+    if archive != 'latest':
         return archive
 
     lock_wait = storage_config.get('lock_wait', None)
@@ -31,7 +36,7 @@ def resolve_archive_name(
         + flags.make_flags('lock-wait', lock_wait)
         + flags.make_flags('last', 1)
         + ('--short',)
-        + flags.make_repository_flags(repository, local_borg_version)
+        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     output = execute_command_and_capture_output(
@@ -42,16 +47,16 @@ def resolve_archive_name(
     except IndexError:
         raise ValueError('No archives found in the repository')
 
-    logger.debug('{}: Latest archive is {}'.format(repository, latest_archive))
+    logger.debug(f'{repository_path}: Latest archive is {latest_archive}')
 
     return latest_archive
 
 
-MAKE_FLAGS_EXCLUDES = ('repository', 'prefix')
+MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
 
 
 def make_rlist_command(
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     rlist_arguments,
@@ -89,15 +94,21 @@ def make_rlist_command(
                 else flags.make_flags('glob-archives', f'{rlist_arguments.prefix}*')
             )
             if rlist_arguments.prefix
-            else ()
+            else (
+                flags.make_match_archives_flags(
+                    rlist_arguments.match_archives or storage_config.get('match_archives'),
+                    storage_config.get('archive_name_format'),
+                    local_borg_version,
+                )
+            )
         )
         + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES)
-        + flags.make_repository_flags(repository, local_borg_version)
+        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
 
 def list_repository(
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     rlist_arguments,
@@ -113,11 +124,16 @@ def list_repository(
     borg_environment = environment.make_environment(storage_config)
 
     main_command = make_rlist_command(
-        repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path
+        repository_path,
+        storage_config,
+        local_borg_version,
+        rlist_arguments,
+        local_path,
+        remote_path,
     )
 
     if rlist_arguments.json:
-        return execute_command_and_capture_output(main_command, extra_environment=borg_environment,)
+        return execute_command_and_capture_output(main_command, extra_environment=borg_environment)
     else:
         execute_command(
             main_command,

borgmatic/borg/transfer.py

@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
 
 def transfer_archives(
     dry_run,
-    repository,
+    repository_path,
     storage_config,
     local_borg_version,
     transfer_arguments,
@@ -28,17 +28,22 @@ def transfer_archives(
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('remote-path', remote_path)
         + flags.make_flags('lock-wait', storage_config.get('lock_wait', None))
-        + (('--progress',) if transfer_arguments.progress else ())
         + (
-            flags.make_flags(
-                'match-archives', transfer_arguments.match_archives or transfer_arguments.archive
+            flags.make_flags_from_arguments(
+                transfer_arguments,
+                excludes=('repository', 'source_repository', 'archive', 'match_archives'),
+            )
+            or (
+                flags.make_match_archives_flags(
+                    transfer_arguments.match_archives
+                    or transfer_arguments.archive
+                    or storage_config.get('match_archives'),
+                    storage_config.get('archive_name_format'),
+                    local_borg_version,
+                )
             )
         )
-        + flags.make_flags_from_arguments(
-            transfer_arguments,
-            excludes=('repository', 'source_repository', 'archive', 'match_archives'),
-        )
-        + flags.make_repository_flags(repository, local_borg_version)
+        + flags.make_repository_flags(repository_path, local_borg_version)
         + flags.make_flags('other-repo', transfer_arguments.source_repository)
         + flags.make_flags('dry-run', dry_run)
     )

borgmatic/commands/arguments.py

@@ -46,11 +46,12 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
     if 'borg' in unparsed_arguments:
         subparsers = {'borg': subparsers['borg']}
 
-    for subparser_name, subparser in subparsers.items():
-        if subparser_name not in remaining_arguments:
-            continue
+    for argument in remaining_arguments:
+        canonical_name = alias_to_subparser_name.get(argument, argument)
+        subparser = subparsers.get(canonical_name)
 
-        canonical_name = alias_to_subparser_name.get(subparser_name, subparser_name)
+        if not subparser:
+            continue
 
         # If a parsed value happens to be the same as the name of a subparser, remove it from the
         # remaining arguments. This prevents, for instance, "check --only extract" from triggering
@@ -67,9 +68,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
 
         arguments[canonical_name] = parsed
 
-    # If no actions are explicitly requested, assume defaults: prune, compact, create, and check.
+    # If no actions are explicitly requested, assume defaults.
     if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
-        for subparser_name in ('prune', 'compact', 'create', 'check'):
+        for subparser_name in ('create', 'prune', 'compact', 'check'):
             subparser = subparsers[subparser_name]
             parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
             arguments[subparser_name] = parsed
@@ -130,9 +131,7 @@ def make_parsers():
         nargs='*',
         dest='config_paths',
         default=config_paths,
-        help='Configuration filenames or directories, defaults to: {}'.format(
-            ' '.join(unexpanded_config_paths)
-        ),
+        help=f"Configuration filenames or directories, defaults to: {' '.join(unexpanded_config_paths)}",
     )
     global_group.add_argument(
         '--excludes',
@@ -179,10 +178,12 @@ def make_parsers():
         help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)',
     )
     global_group.add_argument(
-        '--log-file',
+        '--log-file', type=str, help='Write log messages to this file instead of syslog',
+    )
+    global_group.add_argument(
+        '--log-file-format',
         type=str,
-        default=None,
-        help='Write log messages to this file instead of syslog',
+        help='Log format string used for log messages written to the log file',
     )
     global_group.add_argument(
         '--override',
@@ -215,7 +216,7 @@ def make_parsers():
     top_level_parser = ArgumentParser(
         description='''
             Simple, configuration-driven backup software for servers and workstations. If none of
-            the action options are given, then borgmatic defaults to: prune, compact, create, and
+            the action options are given, then borgmatic defaults to: create, prune, compact, and
             check.
             ''',
         parents=[global_parser],
@@ -224,7 +225,7 @@ def make_parsers():
     subparsers = top_level_parser.add_subparsers(
         title='actions',
         metavar='',
-        help='Specify zero or more actions. Defaults to prune, compact, create, and check. Use --help with action for details:',
+        help='Specify zero or more actions. Defaults to create, prune, compact, and check. Use --help with action for details:',
     )
     rcreate_parser = subparsers.add_parser(
         'rcreate',
@@ -294,7 +295,7 @@ def make_parsers():
     )
     transfer_group.add_argument(
         '--upgrader',
-        help='Upgrader type used to convert the transfered data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion',
+        help='Upgrader type used to convert the transferred data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion',
     )
     transfer_group.add_argument(
         '--progress',
@@ -332,6 +333,10 @@ def make_parsers():
         add_help=False,
     )
     prune_group = prune_parser.add_argument_group('prune arguments')
+    prune_group.add_argument(
+        '--repository',
+        help='Path of specific existing repository to prune (must be already specified in a borgmatic configuration file)',
+    )
     prune_group.add_argument(
         '--stats',
         dest='stats',
@@ -352,6 +357,10 @@ def make_parsers():
         add_help=False,
     )
     compact_group = compact_parser.add_argument_group('compact arguments')
+    compact_group.add_argument(
+        '--repository',
+        help='Path of specific existing repository to compact (must be already specified in a borgmatic configuration file)',
+    )
     compact_group.add_argument(
         '--progress',
         dest='progress',
@@ -384,6 +393,10 @@ def make_parsers():
         add_help=False,
     )
     create_group = create_parser.add_argument_group('create arguments')
+    create_group.add_argument(
+        '--repository',
+        help='Path of specific existing repository to backup to (must be already specified in a borgmatic configuration file)',
+    )
     create_group.add_argument(
         '--progress',
         dest='progress',
@@ -414,6 +427,10 @@ def make_parsers():
         add_help=False,
     )
     check_group = check_parser.add_argument_group('check arguments')
+    check_group.add_argument(
+        '--repository',
+        help='Path of specific existing repository to check (must be already specified in a borgmatic configuration file)',
+    )
     check_group.add_argument(
         '--progress',
         dest='progress',
@@ -475,10 +492,9 @@ def make_parsers():
     )
     extract_group.add_argument(
         '--strip-components',
-        type=int,
+        type=lambda number: number if number == 'all' else int(number),
         metavar='NUMBER',
-        dest='strip_components',
-        help='Number of leading path components to remove from each extracted path. Skip paths with fewer elements',
+        help='Number of leading path components to remove from each extracted path or "all" to strip all leading path components. Skip paths with fewer elements',
     )
     extract_group.add_argument(
         '--progress',
@@ -611,7 +627,7 @@ def make_parsers():
         metavar='NAME',
         nargs='+',
         dest='databases',
-        help='Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic\'s configuration',
+        help="Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic's configuration",
     )
     restore_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
@@ -636,7 +652,7 @@ def make_parsers():
         '--json', default=False, action='store_true', help='Output results as JSON'
     )
     rlist_group.add_argument(
-        '-P', '--prefix', help='Only list archive names starting with this prefix'
+        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
     )
     rlist_group.add_argument(
         '-a',
@@ -691,7 +707,7 @@ def make_parsers():
         '--json', default=False, action='store_true', help='Output results as JSON'
     )
     list_group.add_argument(
-        '-P', '--prefix', help='Only list archive names starting with this prefix'
+        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
     )
     list_group.add_argument(
         '-a',
@@ -763,7 +779,9 @@ def make_parsers():
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )
     info_group.add_argument(
-        '-P', '--prefix', help='Only show info for archive names starting with this prefix'
+        '-P',
+        '--prefix',
+        help='Deprecated. Only show info for archive names starting with this prefix',
     )
     info_group.add_argument(
         '-a',
@@ -805,7 +823,7 @@ def make_parsers():
         'borg',
         aliases=SUBPARSER_ALIASES['borg'],
         help='Run an arbitrary Borg command',
-        description='Run an arbitrary Borg command based on borgmatic\'s configuration',
+        description="Run an arbitrary Borg command based on borgmatic's configuration",
         add_help=False,
     )
     borg_group = borg_parser.add_argument_group('borg arguments')
@@ -861,7 +879,17 @@ def parse_arguments(*unparsed_arguments):
         and arguments['transfer'].match_archives
     ):
         raise ValueError(
-            'With the transfer action, only one of --archive and --glob-archives flags can be used.'
+            'With the transfer action, only one of --archive and --match-archives flags can be used.'
+        )
+
+    if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives):
+        raise ValueError(
+            'With the list action, only one of --prefix or --match-archives flags can be used.'
+        )
+
+    if 'rlist' in arguments and (arguments['rlist'].prefix and arguments['rlist'].match_archives):
+        raise ValueError(
+            'With the rlist action, only one of --prefix or --match-archives flags can be used.'
         )
 
     if 'info' in arguments and (

borgmatic/commands/borgmatic.py

@@ -44,8 +44,8 @@ LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
 def run_configuration(config_filename, config, arguments):
     '''
     Given a config filename, the corresponding parsed config dict, and command-line arguments as a
-    dict from subparser name to a namespace of parsed arguments, execute the defined prune, compact,
-    create, check, and/or other actions.
+    dict from subparser name to a namespace of parsed arguments, execute the defined create, prune,
+    compact, check, and/or other actions.
 
     Yield a combination of:
 
@@ -64,15 +64,13 @@ def run_configuration(config_filename, config, arguments):
     retry_wait = storage.get('retry_wait', 0)
     encountered_error = None
     error_repository = ''
-    using_primary_action = {'prune', 'compact', 'create', 'check'}.intersection(arguments)
+    using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments)
     monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity)
 
     try:
         local_borg_version = borg_version.local_borg_version(storage, local_path)
     except (OSError, CalledProcessError, ValueError) as error:
-        yield from log_error_records(
-            '{}: Error getting local Borg version'.format(config_filename), error
-        )
+        yield from log_error_records(f'{config_filename}: Error getting local Borg version', error)
         return
 
     try:
@@ -100,7 +98,7 @@ def run_configuration(config_filename, config, arguments):
             return
 
         encountered_error = error
-        yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error)
+        yield from log_error_records(f'{config_filename}: Error pinging monitor', error)
 
     if not encountered_error:
         repo_queue = Queue()
@@ -108,7 +106,8 @@ def run_configuration(config_filename, config, arguments):
             repo_queue.put((repo, 0),)
 
         while not repo_queue.empty():
-            repository_path, retry_num = repo_queue.get()
+            repository, retry_num = repo_queue.get()
+            logger.debug(f'{repository["path"]}: Running actions for repository')
             timeout = retry_num * retry_wait
             if timeout:
                 logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry')
@@ -125,14 +124,14 @@ def run_configuration(config_filename, config, arguments):
                     local_path=local_path,
                     remote_path=remote_path,
                     local_borg_version=local_borg_version,
-                    repository_path=repository_path,
+                    repository=repository,
                 )
             except (OSError, CalledProcessError, ValueError) as error:
                 if retry_num < retries:
-                    repo_queue.put((repository_path, retry_num + 1),)
+                    repo_queue.put((repository, retry_num + 1),)
                     tuple(  # Consume the generator so as to trigger logging.
                         log_error_records(
-                            '{}: Error running actions for repository'.format(repository_path),
+                            f'{repository["path"]}: Error running actions for repository',
                             error,
                             levelno=logging.WARNING,
                             log_command_error_output=True,
@@ -147,10 +146,29 @@ def run_configuration(config_filename, config, arguments):
                     return
 
                 yield from log_error_records(
-                    '{}: Error running actions for repository'.format(repository_path), error
+                    f'{repository["path"]}: Error running actions for repository', error
                 )
                 encountered_error = error
-                error_repository = repository_path
+                error_repository = repository['path']
+
+    try:
+        if using_primary_action:
+            # send logs irrespective of error
+            dispatch.call_hooks(
+                'ping_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitor.State.LOG,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
+    except (OSError, CalledProcessError) as error:
+        if command.considered_soft_failure(config_filename, error):
+            return
+
+        encountered_error = error
+        yield from log_error_records(f'{repository["path"]}: Error pinging monitor', error)
 
     if not encountered_error:
         try:
@@ -177,7 +195,7 @@ def run_configuration(config_filename, config, arguments):
                 return
 
             encountered_error = error
-            yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error)
+            yield from log_error_records(f'{config_filename}: Error pinging monitor', error)
 
     if encountered_error and using_primary_action:
         try:
@@ -212,9 +230,7 @@ def run_configuration(config_filename, config, arguments):
             if command.considered_soft_failure(config_filename, error):
                 return
 
-            yield from log_error_records(
-                '{}: Error running on-error hook'.format(config_filename), error
-            )
+            yield from log_error_records(f'{config_filename}: Error running on-error hook', error)
 
 
 def run_actions(
@@ -229,7 +245,7 @@ def run_actions(
     local_path,
     remote_path,
     local_borg_version,
-    repository_path,
+    repository,
 ):
     '''
     Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration
@@ -244,13 +260,14 @@ def run_actions(
     invalid.
     '''
     add_custom_log_levels()
-    repository = os.path.expanduser(repository_path)
+    repository_path = os.path.expanduser(repository['path'])
     global_arguments = arguments['global']
     dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else ''
     hook_context = {
         'repository': repository_path,
         # Deprecated: For backwards compatibility with borgmatic < 1.6.0.
-        'repositories': ','.join(location['repositories']),
+        'repositories': ','.join([repo['path'] for repo in location['repositories']]),
+        'log_file': global_arguments.log_file if global_arguments.log_file else '',
     }
 
     command.execute_hook(
@@ -262,155 +279,162 @@ def run_actions(
         **hook_context,
     )
 
-    if 'rcreate' in arguments:
-        borgmatic.actions.rcreate.run_rcreate(
-            repository,
-            storage,
-            local_borg_version,
-            arguments['rcreate'],
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-    if 'transfer' in arguments:
-        borgmatic.actions.transfer.run_transfer(
-            repository,
-            storage,
-            local_borg_version,
-            arguments['transfer'],
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-    if 'prune' in arguments:
-        borgmatic.actions.prune.run_prune(
-            config_filename,
-            repository,
-            storage,
-            retention,
-            hooks,
-            hook_context,
-            local_borg_version,
-            arguments['prune'],
-            global_arguments,
-            dry_run_label,
-            local_path,
-            remote_path,
-        )
-    if 'compact' in arguments:
-        borgmatic.actions.compact.run_compact(
-            config_filename,
-            repository,
-            storage,
-            retention,
-            hooks,
-            hook_context,
-            local_borg_version,
-            arguments['compact'],
-            global_arguments,
-            dry_run_label,
-            local_path,
-            remote_path,
-        )
-    if 'create' in arguments:
-        yield from borgmatic.actions.create.run_create(
-            config_filename,
-            repository,
-            location,
-            storage,
-            hooks,
-            hook_context,
-            local_borg_version,
-            arguments['create'],
-            global_arguments,
-            dry_run_label,
-            local_path,
-            remote_path,
-        )
-    if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
-        borgmatic.actions.check.run_check(
-            config_filename,
-            repository,
-            location,
-            storage,
-            consistency,
-            hooks,
-            hook_context,
-            local_borg_version,
-            arguments['check'],
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-    if 'extract' in arguments:
-        borgmatic.actions.extract.run_extract(
-            config_filename,
-            repository,
-            location,
-            storage,
-            hooks,
-            hook_context,
-            local_borg_version,
-            arguments['extract'],
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-    if 'export-tar' in arguments:
-        borgmatic.actions.export_tar.run_export_tar(
-            repository,
-            storage,
-            local_borg_version,
-            arguments['export-tar'],
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-    if 'mount' in arguments:
-        borgmatic.actions.mount.run_mount(
-            repository, storage, local_borg_version, arguments['mount'], local_path, remote_path,
-        )
-    if 'restore' in arguments:
-        borgmatic.actions.restore.run_restore(
-            repository,
-            location,
-            storage,
-            hooks,
-            local_borg_version,
-            arguments['restore'],
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-    if 'rlist' in arguments:
-        yield from borgmatic.actions.rlist.run_rlist(
-            repository, storage, local_borg_version, arguments['rlist'], local_path, remote_path,
-        )
-    if 'list' in arguments:
-        yield from borgmatic.actions.list.run_list(
-            repository, storage, local_borg_version, arguments['list'], local_path, remote_path,
-        )
-    if 'rinfo' in arguments:
-        yield from borgmatic.actions.rinfo.run_rinfo(
-            repository, storage, local_borg_version, arguments['rinfo'], local_path, remote_path,
-        )
-    if 'info' in arguments:
-        yield from borgmatic.actions.info.run_info(
-            repository, storage, local_borg_version, arguments['info'], local_path, remote_path,
-        )
-    if 'break-lock' in arguments:
-        borgmatic.actions.break_lock.run_break_lock(
-            repository,
-            storage,
-            local_borg_version,
-            arguments['break-lock'],
-            local_path,
-            remote_path,
-        )
-    if 'borg' in arguments:
-        borgmatic.actions.borg.run_borg(
-            repository, storage, local_borg_version, arguments['borg'], local_path, remote_path,
-        )
+    for (action_name, action_arguments) in arguments.items():
+        if action_name == 'rcreate':
+            borgmatic.actions.rcreate.run_rcreate(
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'transfer':
+            borgmatic.actions.transfer.run_transfer(
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'create':
+            yield from borgmatic.actions.create.run_create(
+                config_filename,
+                repository,
+                location,
+                storage,
+                hooks,
+                hook_context,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                dry_run_label,
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'prune':
+            borgmatic.actions.prune.run_prune(
+                config_filename,
+                repository,
+                storage,
+                retention,
+                hooks,
+                hook_context,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                dry_run_label,
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'compact':
+            borgmatic.actions.compact.run_compact(
+                config_filename,
+                repository,
+                storage,
+                retention,
+                hooks,
+                hook_context,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                dry_run_label,
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'check':
+            if checks.repository_enabled_for_checks(repository, consistency):
+                borgmatic.actions.check.run_check(
+                    config_filename,
+                    repository,
+                    location,
+                    storage,
+                    consistency,
+                    hooks,
+                    hook_context,
+                    local_borg_version,
+                    action_arguments,
+                    global_arguments,
+                    local_path,
+                    remote_path,
+                )
+        elif action_name == 'extract':
+            borgmatic.actions.extract.run_extract(
+                config_filename,
+                repository,
+                location,
+                storage,
+                hooks,
+                hook_context,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'export-tar':
+            borgmatic.actions.export_tar.run_export_tar(
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'mount':
+            borgmatic.actions.mount.run_mount(
+                repository,
+                storage,
+                local_borg_version,
+                arguments['mount'],
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'restore':
+            borgmatic.actions.restore.run_restore(
+                repository,
+                location,
+                storage,
+                hooks,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'rlist':
+            yield from borgmatic.actions.rlist.run_rlist(
+                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+            )
+        elif action_name == 'list':
+            yield from borgmatic.actions.list.run_list(
+                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+            )
+        elif action_name == 'rinfo':
+            yield from borgmatic.actions.rinfo.run_rinfo(
+                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+            )
+        elif action_name == 'info':
+            yield from borgmatic.actions.info.run_info(
+                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+            )
+        elif action_name == 'break-lock':
+            borgmatic.actions.break_lock.run_break_lock(
+                repository,
+                storage,
+                local_borg_version,
+                arguments['break-lock'],
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'borg':
+            borgmatic.actions.borg.run_borg(
+                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+            )
 
     command.execute_hook(
         hooks.get('after_actions'),
@@ -446,9 +470,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
                         dict(
                             levelno=logging.WARNING,
                             levelname='WARNING',
-                            msg='{}: Insufficient permissions to read configuration file'.format(
-                                config_filename
-                            ),
+                            msg=f'{config_filename}: Insufficient permissions to read configuration file',
                         )
                     ),
                 ]
@@ -460,7 +482,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
                         dict(
                             levelno=logging.CRITICAL,
                             levelname='CRITICAL',
-                            msg='{}: Error parsing configuration file'.format(config_filename),
+                            msg=f'{config_filename}: Error parsing configuration file',
                         )
                     ),
                     logging.makeLogRecord(
@@ -561,9 +583,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
 
     if not configs:
         yield from log_error_records(
-            '{}: No valid configuration files found'.format(
-                ' '.join(arguments['global'].config_paths)
-            )
+            f"{' '.join(arguments['global'].config_paths)}: No valid configuration files found",
         )
         return
 
@@ -589,23 +609,21 @@ def collect_configuration_run_summary_logs(configs, arguments):
         error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord))
 
         if error_logs:
-            yield from log_error_records(
-                '{}: Error running configuration file'.format(config_filename)
-            )
+            yield from log_error_records(f'{config_filename}: An error occurred')
             yield from error_logs
         else:
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.INFO,
                     levelname='INFO',
-                    msg='{}: Successfully ran configuration file'.format(config_filename),
+                    msg=f'{config_filename}: Successfully ran configuration file',
                 )
             )
             if results:
                 json_results.extend(results)
 
     if 'umount' in arguments:
-        logger.info('Unmounting mount point {}'.format(arguments['umount'].mount_point))
+        logger.info(f"Unmounting mount point {arguments['umount'].mount_point}")
         try:
             borg_umount.unmount_archive(
                 mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs),
@@ -653,7 +671,7 @@ def main():  # pragma: no cover
         if error.code == 0:
             raise error
         configure_logging(logging.CRITICAL)
-        logger.critical('Error parsing arguments: {}'.format(' '.join(sys.argv)))
+        logger.critical(f"Error parsing arguments: {' '.join(sys.argv)}")
         exit_with_help_link()
 
     global_arguments = arguments['global']
@@ -683,10 +701,11 @@ def main():  # pragma: no cover
             verbosity_to_log_level(global_arguments.log_file_verbosity),
             verbosity_to_log_level(global_arguments.monitoring_verbosity),
             global_arguments.log_file,
+            global_arguments.log_file_format,
         )
     except (FileNotFoundError, PermissionError) as error:
         configure_logging(logging.CRITICAL)
-        logger.critical('Error configuring logging: {}'.format(error))
+        logger.critical(f'Error configuring logging: {error}')
         exit_with_help_link()
 
     logger.debug('Ensuring legacy configuration is upgraded')

borgmatic/commands/completion.py

@@ -34,7 +34,7 @@ def bash_completion():
             '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
             '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',
             '    if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];'
-            '        then cat << EOF\n%s\nEOF' % UPGRADE_MESSAGE,
+            f'        then cat << EOF\n{UPGRADE_MESSAGE}\nEOF',
             '    fi',
             '}',
             'complete_borgmatic() {',
@@ -48,7 +48,7 @@ def bash_completion():
             for action, subparser in subparsers.choices.items()
         )
         + (
-            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'
+            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: FS003
             % (actions, global_flags),
             '    (check_version &)',
             '}',

borgmatic/commands/convert_config.py

@@ -28,9 +28,7 @@ def parse_arguments(*arguments):
         '--source-config',
         dest='source_config_filename',
         default=DEFAULT_SOURCE_CONFIG_FILENAME,
-        help='Source INI-style configuration filename. Default: {}'.format(
-            DEFAULT_SOURCE_CONFIG_FILENAME
-        ),
+        help=f'Source INI-style configuration filename. Default: {DEFAULT_SOURCE_CONFIG_FILENAME}',
     )
     parser.add_argument(
         '-e',
@@ -46,9 +44,7 @@ def parse_arguments(*arguments):
         '--destination-config',
         dest='destination_config_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help='Destination YAML configuration filename. Default: {}'.format(
-            DEFAULT_DESTINATION_CONFIG_FILENAME
-        ),
+        help=f'Destination YAML configuration filename. Default: {DEFAULT_DESTINATION_CONFIG_FILENAME}',
     )
 
     return parser.parse_args(arguments)
@@ -59,19 +55,15 @@ TEXT_WRAP_CHARACTERS = 80
 
 def display_result(args):  # pragma: no cover
     result_lines = textwrap.wrap(
-        'Your borgmatic configuration has been upgraded. Please review the result in {}.'.format(
-            args.destination_config_filename
-        ),
+        f'Your borgmatic configuration has been upgraded. Please review the result in {args.destination_config_filename}.',
         TEXT_WRAP_CHARACTERS,
     )
 
+    excludes_phrase = (
+        f' and {args.source_excludes_filename}' if args.source_excludes_filename else ''
+    )
     delete_lines = textwrap.wrap(
-        'Once you are satisfied, you can safely delete {}{}.'.format(
-            args.source_config_filename,
-            ' and {}'.format(args.source_excludes_filename)
-            if args.source_excludes_filename
-            else '',
-        ),
+        f'Once you are satisfied, you can safely delete {args.source_config_filename}{excludes_phrase}.',
         TEXT_WRAP_CHARACTERS,
     )
 

borgmatic/commands/generate_config.py

@@ -23,9 +23,7 @@ def parse_arguments(*arguments):
         '--destination',
         dest='destination_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help='Destination YAML configuration file, default: {}'.format(
-            DEFAULT_DESTINATION_CONFIG_FILENAME
-        ),
+        help=f'Destination YAML configuration file, default: {DEFAULT_DESTINATION_CONFIG_FILENAME}',
     )
     parser.add_argument(
         '--overwrite',
@@ -48,17 +46,13 @@ def main():  # pragma: no cover
             overwrite=args.overwrite,
         )
 
-        print('Generated a sample configuration file at {}.'.format(args.destination_filename))
+        print(f'Generated a sample configuration file at {args.destination_filename}.')
         print()
         if args.source_filename:
-            print(
-                'Merged in the contents of configuration file at {}.'.format(args.source_filename)
-            )
+            print(f'Merged in the contents of configuration file at {args.source_filename}.')
             print('To review the changes made, run:')
             print()
-            print(
-                '    diff --unified {} {}'.format(args.source_filename, args.destination_filename)
-            )
+            print(f'    diff --unified {args.source_filename} {args.destination_filename}')
             print()
         print('This includes all available configuration options with example values. The few')
         print('required options are indicated. Please edit the file to suit your needs.')

borgmatic/commands/validate_config.py

@@ -2,6 +2,7 @@ import logging
 import sys
 from argparse import ArgumentParser
 
+import borgmatic.config.generate
 from borgmatic.config import collect, validate
 
 logger = logging.getLogger(__name__)
@@ -21,20 +22,24 @@ def parse_arguments(*arguments):
         nargs='+',
         dest='config_paths',
         default=config_paths,
-        help='Configuration filenames or directories, defaults to: {}'.format(
-            ' '.join(config_paths)
-        ),
+        help=f'Configuration filenames or directories, defaults to: {config_paths}',
+    )
+    parser.add_argument(
+        '-s',
+        '--show',
+        action='store_true',
+        help='Show the validated configuration after all include merging has occurred',
     )
 
     return parser.parse_args(arguments)
 
 
 def main():  # pragma: no cover
-    args = parse_arguments(*sys.argv[1:])
+    arguments = parse_arguments(*sys.argv[1:])
 
     logging.basicConfig(level=logging.INFO, format='%(message)s')
 
-    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths))
     if len(config_filenames) == 0:
         logger.critical('No files to validate found')
         sys.exit(1)
@@ -42,15 +47,22 @@ def main():  # pragma: no cover
     found_issues = False
     for config_filename in config_filenames:
         try:
-            validate.parse_configuration(config_filename, validate.schema_filename())
+            config, parse_logs = validate.parse_configuration(
+                config_filename, validate.schema_filename()
+            )
         except (ValueError, OSError, validate.Validation_error) as error:
-            logging.critical('{}: Error parsing configuration file'.format(config_filename))
+            logging.critical(f'{config_filename}: Error parsing configuration file')
             logging.critical(error)
             found_issues = True
+        else:
+            for log in parse_logs:
+                logger.handle(log)
+
+            if arguments.show:
+                print('---')
+                print(borgmatic.config.generate.render_configuration(config))
 
     if found_issues:
         sys.exit(1)
-    else:
-        logger.info(
-            'All given configuration files are valid: {}'.format(', '.join(config_filenames))
-        )
+
+    logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}")

borgmatic/config/collect.py

@@ -16,8 +16,8 @@ def get_default_config_paths(expand_home=True):
     return [
         '/etc/borgmatic/config.yaml',
         '/etc/borgmatic.d',
-        '%s/borgmatic/config.yaml' % user_config_directory,
-        '%s/borgmatic.d' % user_config_directory,
+        os.path.join(user_config_directory, 'borgmatic/config.yaml'),
+        os.path.join(user_config_directory, 'borgmatic.d'),
     ]
 
 

borgmatic/config/convert.py

@@ -43,7 +43,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         ]
     )
 
-    # Split space-seperated values into actual lists, make "repository" into a list, and merge in
+    # Split space-separated values into actual lists, make "repository" into a list, and merge in
     # excludes.
     location = destination_config['location']
     location['source_directories'] = source_config.location['source_directories'].split(' ')

borgmatic/config/environment.py

@@ -14,11 +14,14 @@ def _resolve_string(matcher):
     if matcher.group('escape') is not None:
         # in case of escaped envvar, unescape it
         return matcher.group('variable')
+
     # resolve the env var
     name, default = matcher.group('name'), matcher.group('default')
     out = os.getenv(name, default=default)
+
     if out is None:
-        raise ValueError('Cannot find variable ${name} in environment'.format(name=name))
+        raise ValueError(f'Cannot find variable {name} in environment')
+
     return out
 
 

borgmatic/config/generate.py

@@ -48,7 +48,7 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
             config, schema, indent=indent, skip_first=parent_is_sequence
         )
     else:
-        raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema))
+        raise ValueError(f'Schema at level {level} is unsupported: {schema}')
 
     return config
 
@@ -84,7 +84,7 @@ def _comment_out_optional_configuration(rendered_config):
     for line in rendered_config.split('\n'):
         # Upon encountering an optional configuration option, comment out lines until the next blank
         # line.
-        if line.strip().startswith('# {}'.format(COMMENTED_OUT_SENTINEL)):
+        if line.strip().startswith(f'# {COMMENTED_OUT_SENTINEL}'):
             optional = True
             continue
 
@@ -117,9 +117,7 @@ def write_configuration(config_filename, rendered_config, mode=0o600, overwrite=
     '''
     if not overwrite and os.path.exists(config_filename):
         raise FileExistsError(
-            '{} already exists. Aborting. Use --overwrite to replace the file.'.format(
-                config_filename
-            )
+            f'{config_filename} already exists. Aborting. Use --overwrite to replace the file.'
         )
 
     try:
@@ -218,7 +216,7 @@ def remove_commented_out_sentinel(config, field_name):
     except KeyError:
         return
 
-    if last_comment_value == '# {}\n'.format(COMMENTED_OUT_SENTINEL):
+    if last_comment_value == f'# {COMMENTED_OUT_SENTINEL}\n':
         config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX].pop()
 
 

borgmatic/config/legacy.py

@@ -70,13 +70,11 @@ def validate_configuration_format(parser, config_format):
         section_format.name for section_format in config_format
     )
     if unknown_section_names:
-        raise ValueError(
-            'Unknown config sections found: {}'.format(', '.join(unknown_section_names))
-        )
+        raise ValueError(f"Unknown config sections found: {', '.join(unknown_section_names)}")
 
     missing_section_names = set(required_section_names) - section_names
     if missing_section_names:
-        raise ValueError('Missing config sections: {}'.format(', '.join(missing_section_names)))
+        raise ValueError(f"Missing config sections: {', '.join(missing_section_names)}")
 
     for section_format in config_format:
         if section_format.name not in section_names:
@@ -91,9 +89,7 @@ def validate_configuration_format(parser, config_format):
 
         if unexpected_option_names:
             raise ValueError(
-                'Unexpected options found in config section {}: {}'.format(
-                    section_format.name, ', '.join(sorted(unexpected_option_names))
-                )
+                f"Unexpected options found in config section {section_format.name}: {', '.join(sorted(unexpected_option_names))}",
             )
 
         missing_option_names = tuple(
@@ -105,9 +101,7 @@ def validate_configuration_format(parser, config_format):
 
         if missing_option_names:
             raise ValueError(
-                'Required options missing from config section {}: {}'.format(
-                    section_format.name, ', '.join(missing_option_names)
-                )
+                f"Required options missing from config section {section_format.name}: {', '.join(missing_option_names)}",
             )
 
 
@@ -137,7 +131,7 @@ def parse_configuration(config_filename, config_format):
     '''
     parser = RawConfigParser()
     if not parser.read(config_filename):
-        raise ValueError('Configuration file cannot be opened: {}'.format(config_filename))
+        raise ValueError(f'Configuration file cannot be opened: {config_filename}')
 
     validate_configuration_format(parser, config_format)
 

borgmatic/config/load.py

@@ -1,4 +1,5 @@
 import functools
+import json
 import logging
 import os
 
@@ -37,6 +38,24 @@ def include_configuration(loader, filename_node, include_directory):
     return load_configuration(include_filename)
 
 
+def retain_node_error(loader, node):
+    '''
+    Given a ruamel.yaml.loader.Loader and a YAML node, raise an error.
+
+    Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was
+    used in a configuration file without a merge. In configuration files with a merge, mapping and
+    sequence nodes with "!retain" tags are handled by deep_merge_nodes() below.
+
+    Also raise ValueError if a scalar node is given, as "!retain" is not supported on scalar nodes.
+    '''
+    if isinstance(node, (ruamel.yaml.nodes.MappingNode, ruamel.yaml.nodes.SequenceNode)):
+        raise ValueError(
+            'The !retain tag may only be used within a configuration file containing a merged !include tag.'
+        )
+
+    raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.')
+
+
 class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
     A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
@@ -49,6 +68,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
             '!include',
             functools.partial(include_configuration, include_directory=include_directory),
         )
+        self.add_constructor('!retain', retain_node_error)
 
     def flatten_mapping(self, node):
         '''
@@ -81,7 +101,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
 def load_configuration(filename):
     '''
     Load the given configuration file and return its contents as a data structure of nested dicts
-    and lists.
+    and lists. Also, replace any "{constant}" strings with the value of the "constant" key in the
+    "constants" section of the configuration file.
 
     Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
     if there are too many recursive includes.
@@ -98,7 +119,19 @@ def load_configuration(filename):
     yaml = ruamel.yaml.YAML(typ='safe')
     yaml.Constructor = Include_constructor_with_include_directory
 
-    return yaml.load(open(filename))
+    with open(filename) as file:
+        file_contents = file.read()
+        config = yaml.load(file_contents)
+
+        if config and 'constants' in config:
+            for key, value in config['constants'].items():
+                value = json.dumps(value)
+                file_contents = file_contents.replace(f'{{{key}}}', value.strip('"'))
+
+            config = yaml.load(file_contents)
+            del config['constants']
+
+        return config
 
 
 DELETED_NODE = object()
@@ -162,6 +195,8 @@ def deep_merge_nodes(nodes):
             ),
         ]
 
+    If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged.
+
     The purpose of deep merging like this is to support, for instance, merging one borgmatic
     configuration file into another for reuse, such that a configuration section ("retention",
     etc.) does not completely replace the corresponding section in a merged file.
@@ -184,32 +219,42 @@ def deep_merge_nodes(nodes):
 
                 # If we're dealing with MappingNodes, recurse and merge its values as well.
                 if isinstance(b_value, ruamel.yaml.nodes.MappingNode):
-                    replaced_nodes[(b_key, b_value)] = (
-                        b_key,
-                        ruamel.yaml.nodes.MappingNode(
-                            tag=b_value.tag,
-                            value=deep_merge_nodes(a_value.value + b_value.value),
-                            start_mark=b_value.start_mark,
-                            end_mark=b_value.end_mark,
-                            flow_style=b_value.flow_style,
-                            comment=b_value.comment,
-                            anchor=b_value.anchor,
-                        ),
-                    )
+                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
+                    # downstream schema validation doesn't break on our application-specific tag.
+                    if b_value.tag == '!retain':
+                        b_value.tag = 'tag:yaml.org,2002:map'
+                    else:
+                        replaced_nodes[(b_key, b_value)] = (
+                            b_key,
+                            ruamel.yaml.nodes.MappingNode(
+                                tag=b_value.tag,
+                                value=deep_merge_nodes(a_value.value + b_value.value),
+                                start_mark=b_value.start_mark,
+                                end_mark=b_value.end_mark,
+                                flow_style=b_value.flow_style,
+                                comment=b_value.comment,
+                                anchor=b_value.anchor,
+                            ),
+                        )
                 # If we're dealing with SequenceNodes, merge by appending one sequence to the other.
                 elif isinstance(b_value, ruamel.yaml.nodes.SequenceNode):
-                    replaced_nodes[(b_key, b_value)] = (
-                        b_key,
-                        ruamel.yaml.nodes.SequenceNode(
-                            tag=b_value.tag,
-                            value=a_value.value + b_value.value,
-                            start_mark=b_value.start_mark,
-                            end_mark=b_value.end_mark,
-                            flow_style=b_value.flow_style,
-                            comment=b_value.comment,
-                            anchor=b_value.anchor,
-                        ),
-                    )
+                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
+                    # downstream schema validation doesn't break on our application-specific tag.
+                    if b_value.tag == '!retain':
+                        b_value.tag = 'tag:yaml.org,2002:seq'
+                    else:
+                        replaced_nodes[(b_key, b_value)] = (
+                            b_key,
+                            ruamel.yaml.nodes.SequenceNode(
+                                tag=b_value.tag,
+                                value=a_value.value + b_value.value,
+                                start_mark=b_value.start_mark,
+                                end_mark=b_value.end_mark,
+                                flow_style=b_value.flow_style,
+                                comment=b_value.comment,
+                                anchor=b_value.anchor,
+                            ),
+                        )
 
     return [
         replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE

borgmatic/config/normalize.py

@@ -1,4 +1,5 @@
 import logging
+import os
 
 
 def normalize(config_filename, config):
@@ -56,9 +57,15 @@ def normalize(config_filename, config):
     # Upgrade remote repositories to ssh:// syntax, required in Borg 2.
     repositories = location.get('repositories')
     if repositories:
+        if isinstance(repositories[0], str):
+            config['location']['repositories'] = [
+                {'path': repository} for repository in repositories
+            ]
+            repositories = config['location']['repositories']
         config['location']['repositories'] = []
-        for repository in repositories:
-            if '~' in repository:
+        for repository_dict in repositories:
+            repository_path = repository_dict['path']
+            if '~' in repository_path:
                 logs.append(
                     logging.makeLogRecord(
                         dict(
@@ -68,21 +75,31 @@ def normalize(config_filename, config):
                         )
                     )
                 )
-            if ':' in repository and not repository.startswith('ssh://'):
-                rewritten_repository = (
-                    f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}"
-                )
-                logs.append(
-                    logging.makeLogRecord(
-                        dict(
-                            levelno=logging.WARNING,
-                            levelname='WARNING',
-                            msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository}" as "{rewritten_repository}"',
+            if ':' in repository_path:
+                if repository_path.startswith('file://'):
+                    updated_repository_path = os.path.abspath(
+                        repository_path.partition('file://')[-1]
+                    )
+                    config['location']['repositories'].append(
+                        dict(repository_dict, path=updated_repository_path,)
+                    )
+                elif repository_path.startswith('ssh://'):
+                    config['location']['repositories'].append(repository_dict)
+                else:
+                    rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}"
+                    logs.append(
+                        logging.makeLogRecord(
+                            dict(
+                                levelno=logging.WARNING,
+                                levelname='WARNING',
+                                msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository_path}" as "{rewritten_repository_path}"',
+                            )
                         )
                     )
-                )
-                config['location']['repositories'].append(rewritten_repository)
+                    config['location']['repositories'].append(
+                        dict(repository_dict, path=rewritten_repository_path,)
+                    )
             else:
-                config['location']['repositories'].append(repository)
+                config['location']['repositories'].append(repository_dict)
 
     return logs

borgmatic/config/schema.yaml

@@ -3,6 +3,17 @@ required:
     - location
 additionalProperties: false
 properties:
+    constants:
+        type: object
+        description: |
+            Constants to use in the configuration file. All occurrences of the
+            constant name within culy braces will be replaced with the value.
+            For example, if you have a constant named "hostname" with the value
+            "myhostname", then the string "{hostname}" will be replaced with
+            "myhostname" in the configuration file.
+        example:
+            hostname: myhostname
+            prefix: myprefix
     location:
         type: object
         description: |
@@ -29,19 +40,32 @@ properties:
             repositories:
                 type: array
                 items:
-                    type: string
+                    type: object 
+                    required:
+                        - path
+                    properties:
+                        path:
+                            type: string
+                            example: ssh://user@backupserver/./{fqdn}
+                        label:
+                            type: string
+                            example: backupserver
                 description: |
-                    Paths to local or remote repositories (required). Tildes are
-                    expanded. Multiple repositories are backed up to in
-                    sequence. Borg placeholders can be used. See the output of
-                    "borg help placeholders" for details. See ssh_command for
-                    SSH options like identity file or port. If systemd service
-                    is used, then add local repository paths in the systemd
-                    service file to the ReadWritePaths list.
+                    A required list of local or remote repositories with paths
+                    and optional labels (which can be used with the --repository
+                    flag to select a repository). Tildes are expanded. Multiple
+                    repositories are backed up to in sequence. Borg placeholders
+                    can be used. See the output of "borg help placeholders" for
+                    details. See ssh_command for SSH options like identity file
+                    or port. If systemd service is used, then add local
+                    repository paths in the systemd service file to the
+                    ReadWritePaths list. Prior to borgmatic 1.7.10, repositories
+                    was just a list of plain path strings.
                 example:
-                    - ssh://user@backupserver/./sourcehostname.borg
-                    - ssh://user@backupserver/./{fqdn}
-                    - /var/local/backups/local.borg
+                    - path: ssh://user@backupserver/./sourcehostname.borg
+                      label: backupserver
+                    - path: /mnt/backup
+                      label: local
             working_directory:
                 type: string
                 description: |
@@ -202,6 +226,12 @@ properties:
                     path prevents "borgmatic restore" from finding any database
                     dumps created before the change. Defaults to ~/.borgmatic
                 example: /tmp/borgmatic
+            source_directories_must_exist:
+                type: boolean
+                description: |
+                    If true, then source directories must exist, otherwise an
+                    error is raised. Defaults to false.
+                example: true
     storage:
         type: object
         description: |
@@ -315,6 +345,12 @@ properties:
                     Path for Borg cache files. Defaults to
                     $borg_base_directory/.cache/borg
                 example: /path/to/base/cache
+            borg_files_cache_ttl:
+                type: integer
+                description: |
+                    Maximum time to live (ttl) for entries in the Borg files
+                    cache.
+                example: 20
             borg_security_directory:
                 type: string
                 description: |
@@ -342,12 +378,21 @@ properties:
                 description: |
                     Name of the archive. Borg placeholders can be used. See the
                     output of "borg help placeholders" for details. Defaults to
-                    "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this
-                    option, consider also specifying a prefix in the retention
-                    and consistency sections to avoid accidental
-                    pruning/checking of archives with different archive name
-                    formats.
+                    "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". When running
+                    actions like rlist, info, or check, borgmatic automatically
+                    tries to match only archives created with this name format.
                 example: "{hostname}-documents-{now}"
+            match_archives:
+                type: string
+                description: |
+                    A Borg pattern for filtering down the archives used by
+                    borgmatic actions that operate on multiple archives. For
+                    Borg 1.x, use a shell pattern here and see the output of
+                    "borg help placeholders" for details. For Borg 2.x, see the
+                    output of "borg help match-archives". If match_archives is
+                    not specified, borgmatic defaults to deriving the
+                    match_archives value from archive_name_format.
+                example: "sh:{hostname}-*"
             relocated_repo_access_is_ok:
                 type: boolean
                 description: |
@@ -369,6 +414,11 @@ properties:
                         description: |
                           Extra command-line options to pass to "borg init".
                         example: "--extra-option"
+                    create:
+                        type: string
+                        description: |
+                          Extra command-line options to pass to "borg create".
+                        example: "--extra-option"
                     prune:
                         type: string
                         description: |
@@ -379,11 +429,6 @@ properties:
                         description: |
                           Extra command-line options to pass to "borg compact".
                         example: "--extra-option"
-                    create:
-                        type: string
-                        description: |
-                          Extra command-line options to pass to "borg create".
-                        example: "--extra-option"
                     check:
                         type: string
                         description: |
@@ -441,10 +486,12 @@ properties:
             prefix:
                 type: string
                 description: |
-                    When pruning, only consider archive names starting with this
-                    prefix.  Borg placeholders can be used. See the output of
-                    "borg help placeholders" for details. Defaults to
-                    "{hostname}-". Use an empty value to disable the default.
+                    Deprecated. When pruning, only consider archive names
+                    starting with this prefix. Borg placeholders can be used.
+                    See the output of "borg help placeholders" for details.
+                    If a prefix is not specified, borgmatic defaults to
+                    matching archives based on the archive_name_format (see
+                    above).
                 example: sourcehostname
     consistency:
         type: object
@@ -502,12 +549,12 @@ properties:
                 items:
                     type: string
                 description: |
-                    Paths to a subset of the repositories in the location
-                    section on which to run consistency checks. Handy in case
-                    some of your repositories are very large, and so running
-                    consistency checks on them would take too long. Defaults to
-                    running consistency checks on all repositories configured in
-                    the location section.
+                    Paths or labels for a subset of the repositories in the
+                    location section on which to run consistency checks. Handy
+                    in case some of your repositories are very large, and so
+                    running consistency checks on them would take too long.
+                    Defaults to running consistency checks on all repositories
+                    configured in the location section.
                 example:
                     - user@backupserver:sourcehostname.borg
             check_last:
@@ -520,11 +567,12 @@ properties:
             prefix:
                 type: string
                 description: |
-                    When performing the "archives" check, only consider archive
-                    names starting with this prefix. Borg placeholders can be
-                    used. See the output of "borg help placeholders" for
-                    details. Defaults to "{hostname}-". Use an empty value to
-                    disable the default.
+                    Deprecated. When performing the "archives" check, only
+                    consider archive names starting with this prefix. Borg
+                    placeholders can be used. See the output of "borg help
+                    placeholders" for details. If a prefix is not specified,
+                    borgmatic defaults to matching archives based on the
+                    archive_name_format (see above).
                 example: sourcehostname
     output:
         type: object
@@ -663,11 +711,11 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    when an exception occurs during a "prune", "compact",
-                    "create", or "check" action or an associated before/after
+                    when an exception occurs during a "create", "prune",
+                    "compact", or "check" action or an associated before/after
                     hook.
                 example:
-                    - echo "Error during prune/compact/create/check."
+                    - echo "Error during create/prune/compact/check."
             before_everything:
                 type: array
                 items:
@@ -893,14 +941,14 @@ properties:
                             type: string
                             enum: ['sql']
                             description: |
-                                Database dump output format. Currenly only "sql"
-                                is supported. Defaults to "sql" for a single
-                                database. Or, when database name is "all" and
-                                format is blank, dumps all databases to a single
-                                file. But if a format is specified with an "all"
-                                database name, dumps each database to a separate
-                                file of that format, allowing more convenient
-                                restores of individual databases.
+                                Database dump output format. Currently only
+                                "sql" is supported. Defaults to "sql" for a
+                                single database. Or, when database name is "all"
+                                and format is blank, dumps all databases to a
+                                single file. But if a format is specified with
+                                an "all" database name, dumps each database to a
+                                separate file of that format, allowing more
+                                convenient restores of individual databases.
                             example: directory
                         add_drop_database:
                             type: boolean
@@ -941,6 +989,31 @@ properties:
                     mysqldump/mysql commands (from either MySQL or MariaDB). See
                     https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
                     https://mariadb.com/kb/en/library/mysqldump/ for details.
+            sqlite_databases:
+                type: array
+                items:
+                    type: object
+                    required: ['path','name']
+                    additionalProperties: false
+                    properties:
+                        name:
+                            type: string
+                            description: |
+                                This is used to tag the database dump file
+                                with a name. It is not the path to the database
+                                file itself. The name "all" has no special
+                                meaning for SQLite databases.
+                            example: users
+                        path:
+                            type: string
+                            description: |
+                                Path to the SQLite database file to dump. If
+                                relative, it is relative to the current working
+                                directory. Note that using this
+                                database hook implicitly enables both
+                                read_special and one_file_system (see above) to
+                                support dump and restore streaming.
+                            example: /var/lib/sqlite/users.db
             mongodb_databases:
                 type: array
                 items:
@@ -1143,7 +1216,7 @@ properties:
                         type: string
                         description: |
                             Healthchecks ping URL or UUID to notify when a
-                            backup begins, ends, or errors.
+                            backup begins, ends, errors or just to send logs.
                         example: https://hc-ping.com/your-uuid-here
                     verify_tls:
                         type: boolean
@@ -1155,7 +1228,8 @@ properties:
                         type: boolean
                         description: |
                             Send borgmatic logs to Healthchecks as part the
-                            "finish" state. Defaults to true.
+                            "finish", "fail", and "log" states. Defaults to
+                            true.
                         example: false
                     ping_body_limit:
                         type: integer
@@ -1174,10 +1248,11 @@ properties:
                                 - start
                                 - finish
                                 - fail
+                                - log
                             uniqueItems: true
                         description: |
                             List of one or more monitoring states to ping for:
-                            "start", "finish", and/or "fail". Defaults to
+                            "start", "finish", "fail", and/or "log". Defaults to
                             pinging for all states.
                         example:
                             - finish

borgmatic/config/validate.py

@@ -20,9 +20,9 @@ def format_json_error_path_element(path_element):
     Given a path element into a JSON data structure, format it for display as a string.
     '''
     if isinstance(path_element, int):
-        return str('[{}]'.format(path_element))
+        return str(f'[{path_element}]')
 
-    return str('.{}'.format(path_element))
+    return str(f'.{path_element}')
 
 
 def format_json_error(error):
@@ -30,10 +30,10 @@ def format_json_error(error):
     Given an instance of jsonschema.exceptions.ValidationError, format it for display as a string.
     '''
     if not error.path:
-        return 'At the top level: {}'.format(error.message)
+        return f'At the top level: {error.message}'
 
     formatted_path = ''.join(format_json_error_path_element(element) for element in error.path)
-    return "At '{}': {}".format(formatted_path.lstrip('.'), error.message)
+    return f"At '{formatted_path.lstrip('.')}': {error.message}"
 
 
 class Validation_error(ValueError):
@@ -54,9 +54,10 @@ class Validation_error(ValueError):
         '''
         Render a validation error as a user-facing string.
         '''
-        return 'An error occurred while parsing a configuration file at {}:\n'.format(
-            self.config_filename
-        ) + '\n'.join(error for error in self.errors)
+        return (
+            f'An error occurred while parsing a configuration file at {self.config_filename}:\n'
+            + '\n'.join(error for error in self.errors)
+        )
 
 
 def apply_logical_validation(config_filename, parsed_configuration):
@@ -68,13 +69,14 @@ def apply_logical_validation(config_filename, parsed_configuration):
     location_repositories = parsed_configuration.get('location', {}).get('repositories')
     check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', [])
     for repository in check_repositories:
-        if repository not in location_repositories:
+        if not any(
+            repositories_match(repository, config_repository)
+            for config_repository in location_repositories
+        ):
             raise Validation_error(
                 config_filename,
                 (
-                    'Unknown repository in the "consistency" section\'s "check_repositories": {}'.format(
-                        repository
-                    ),
+                    f'Unknown repository in the "consistency" section\'s "check_repositories": {repository}',
                 ),
             )
 
@@ -126,18 +128,29 @@ def normalize_repository_path(repository):
     '''
     Given a repository path, return the absolute path of it (for local repositories).
     '''
-    # A colon in the repository indicates it's a remote repository. Bail.
-    if ':' in repository:
+    # A colon in the repository could mean that it's either a file:// URL or a remote repository.
+    # If it's a remote repository, we don't want to normalize it. If it's a file:// URL, we do.
+    if ':' not in repository:
+        return os.path.abspath(repository)
+    elif repository.startswith('file://'):
+        return os.path.abspath(repository.partition('file://')[-1])
+    else:
         return repository
 
-    return os.path.abspath(repository)
-
 
 def repositories_match(first, second):
     '''
-    Given two repository paths (relative and/or absolute), return whether they match.
+    Given two repository dicts with keys 'path' (relative and/or absolute),
+    and 'label', or two repository paths, return whether they match.
     '''
-    return normalize_repository_path(first) == normalize_repository_path(second)
+    if isinstance(first, str):
+        first = {'path': first, 'label': first}
+    if isinstance(second, str):
+        second = {'path': second, 'label': second}
+    return (first.get('label') == second.get('label')) or (
+        normalize_repository_path(first.get('path'))
+        == normalize_repository_path(second.get('path'))
+    )
 
 
 def guard_configuration_contains_repository(repository, configurations):
@@ -157,14 +170,14 @@ def guard_configuration_contains_repository(repository, configurations):
             config_repository
             for config in configurations.values()
             for config_repository in config['location']['repositories']
-            if repositories_match(repository, config_repository)
+            if repositories_match(config_repository, repository)
         )
     )
 
     if count == 0:
-        raise ValueError('Repository {} not found in configuration files'.format(repository))
+        raise ValueError(f'Repository {repository} not found in configuration files')
     if count > 1:
-        raise ValueError('Repository {} found in multiple configuration files'.format(repository))
+        raise ValueError(f'Repository {repository} found in multiple configuration files')
 
 
 def guard_single_repository_selected(repository, configurations):
@@ -186,5 +199,5 @@ def guard_single_repository_selected(repository, configurations):
 
     if count != 1:
         raise ValueError(
-            'Can\'t determine which repository to use. Use --repository to disambiguate'
+            "Can't determine which repository to use. Use --repository to disambiguate"
         )

borgmatic/execute.py

@@ -11,7 +11,7 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25
 BORG_ERROR_EXIT_CODE = 2
 
 
-def exit_code_indicates_error(process, exit_code, borg_local_path=None):
+def exit_code_indicates_error(command, exit_code, borg_local_path=None):
     '''
     Return True if the given exit code from running a command corresponds to an error. If a Borg
     local path is given and matches the process' command, then treat exit code 1 as a warning
@@ -20,8 +20,6 @@ def exit_code_indicates_error(process, exit_code, borg_local_path=None):
     if exit_code is None:
         return False
 
-    command = process.args.split(' ') if isinstance(process.args, str) else process.args
-
     if borg_local_path and command[0] == borg_local_path:
         return bool(exit_code < 0 or exit_code >= BORG_ERROR_EXIT_CODE)
 
@@ -45,6 +43,23 @@ def output_buffer_for_process(process, exclude_stdouts):
     return process.stderr if process.stdout in exclude_stdouts else process.stdout
 
 
+def append_last_lines(last_lines, captured_output, line, output_log_level):
+    '''
+    Given a rolling list of last lines, a list of captured output, a line to append, and an output
+    log level, append the line to the last lines and (if necessary) the captured output. Then log
+    the line at the requested output log level.
+    '''
+    last_lines.append(line)
+
+    if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+        last_lines.pop(0)
+
+    if output_log_level is None:
+        captured_output.append(line)
+    else:
+        logger.log(output_log_level, line)
+
+
 def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
     '''
     Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
@@ -100,15 +115,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
 
                     # Keep the last few lines of output in case the process errors, and we need the output for
                     # the exception below.
-                    last_lines = buffer_last_lines[ready_buffer]
-                    last_lines.append(line)
-                    if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
-                        last_lines.pop(0)
-
-                    if output_log_level is None:
-                        captured_outputs[ready_process].append(line)
-                    else:
-                        logger.log(output_log_level, line)
+                    append_last_lines(
+                        buffer_last_lines[ready_buffer],
+                        captured_outputs[ready_process],
+                        line,
+                        output_log_level,
+                    )
 
         if not still_running:
             break
@@ -121,13 +133,24 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
             if exit_code is None:
                 still_running = True
 
+            command = process.args.split(' ') if isinstance(process.args, str) else process.args
             # If any process errors, then raise accordingly.
-            if exit_code_indicates_error(process, exit_code, borg_local_path):
+            if exit_code_indicates_error(command, exit_code, borg_local_path):
                 # If an error occurs, include its output in the raised exception so that we don't
                 # inadvertently hide error output.
                 output_buffer = output_buffer_for_process(process, exclude_stdouts)
-
                 last_lines = buffer_last_lines[output_buffer] if output_buffer else []
+
+                # Collect any straggling output lines that came in since we last gathered output.
+                while output_buffer:  # pragma: no cover
+                    line = output_buffer.readline().rstrip().decode()
+                    if not line:
+                        break
+
+                    append_last_lines(
+                        last_lines, captured_outputs[process], line, output_log_level=logging.ERROR
+                    )
+
                 if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
                     last_lines.insert(0, '...')
 
@@ -155,8 +178,8 @@ def log_command(full_command, input_file=None, output_file=None):
     '''
     logger.debug(
         ' '.join(full_command)
-        + (' < {}'.format(getattr(input_file, 'name', '')) if input_file else '')
-        + (' > {}'.format(getattr(output_file, 'name', '')) if output_file else '')
+        + (f" < {getattr(input_file, 'name', '')}" if input_file else '')
+        + (f" > {getattr(output_file, 'name', '')}" if output_file else '')
     )
 
 
@@ -228,13 +251,18 @@ def execute_command_and_capture_output(
     environment = {**os.environ, **extra_environment} if extra_environment else None
     command = ' '.join(full_command) if shell else full_command
 
-    output = subprocess.check_output(
-        command,
-        stderr=subprocess.STDOUT if capture_stderr else None,
-        shell=shell,
-        env=environment,
-        cwd=working_directory,
-    )
+    try:
+        output = subprocess.check_output(
+            command,
+            stderr=subprocess.STDOUT if capture_stderr else None,
+            shell=shell,
+            env=environment,
+            cwd=working_directory,
+        )
+    except subprocess.CalledProcessError as error:
+        if exit_code_indicates_error(command, error.returncode):
+            raise
+        output = error.output
 
     return output.decode() if output is not None else None
 

borgmatic/hooks/command.py

@@ -16,7 +16,7 @@ def interpolate_context(config_filename, hook_description, command, context):
     names/values, interpolate the values by "{name}" into the command and return the result.
     '''
     for name, value in context.items():
-        command = command.replace('{%s}' % name, str(value))
+        command = command.replace(f'{{{name}}}', str(value))
 
     for unsupported_variable in re.findall(r'{\w+}', command):
         logger.warning(
@@ -38,7 +38,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
     Raise subprocesses.CalledProcessError if an error occurs in a hook.
     '''
     if not commands:
-        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
+        logger.debug(f'{config_filename}: No commands to run for {description} hook')
         return
 
     dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
@@ -49,19 +49,15 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
     ]
 
     if len(commands) == 1:
-        logger.info(
-            '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label)
-        )
+        logger.info(f'{config_filename}: Running command for {description} hook{dry_run_label}')
     else:
         logger.info(
-            '{}: Running {} commands for {} hook{}'.format(
-                config_filename, len(commands), description, dry_run_label
-            )
+            f'{config_filename}: Running {len(commands)} commands for {description} hook{dry_run_label}',
         )
 
     if umask:
         parsed_umask = int(str(umask), 8)
-        logger.debug('{}: Set hook umask to {}'.format(config_filename, oct(parsed_umask)))
+        logger.debug(f'{config_filename}: Set hook umask to {oct(parsed_umask)}')
         original_umask = os.umask(parsed_umask)
     else:
         original_umask = None
@@ -93,9 +89,7 @@ def considered_soft_failure(config_filename, error):
 
     if exit_code == SOFT_FAIL_EXIT_CODE:
         logger.info(
-            '{}: Command hook exited with soft failure exit code ({}); skipping remaining actions'.format(
-                config_filename, SOFT_FAIL_EXIT_CODE
-            )
+            f'{config_filename}: Command hook exited with soft failure exit code ({SOFT_FAIL_EXIT_CODE}); skipping remaining actions',
         )
         return True
 

borgmatic/hooks/cronhub.py

@@ -27,18 +27,22 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
     Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration
     filename in any log entries. If this is a dry run, then don't actually ping anything.
     '''
+    if state not in MONITOR_STATE_TO_CRONHUB:
+        logger.debug(
+            f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in Cronhub hook'
+        )
+        return
+
     dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
-    formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state])
+    formatted_state = f'/{MONITOR_STATE_TO_CRONHUB[state]}/'
     ping_url = (
         hook_config['ping_url']
         .replace('/start/', formatted_state)
         .replace('/ping/', formatted_state)
     )
 
-    logger.info(
-        '{}: Pinging Cronhub {}{}'.format(config_filename, state.name.lower(), dry_run_label)
-    )
-    logger.debug('{}: Using Cronhub ping URL {}'.format(config_filename, ping_url))
+    logger.info(f'{config_filename}: Pinging Cronhub {state.name.lower()}{dry_run_label}')
+    logger.debug(f'{config_filename}: Using Cronhub ping URL {ping_url}')
 
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)

borgmatic/hooks/cronitor.py

@@ -27,13 +27,17 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
     Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration
     filename in any log entries. If this is a dry run, then don't actually ping anything.
     '''
-    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
-    ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state])
+    if state not in MONITOR_STATE_TO_CRONITOR:
+        logger.debug(
+            f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in Cronitor hook'
+        )
+        return
 
-    logger.info(
-        '{}: Pinging Cronitor {}{}'.format(config_filename, state.name.lower(), dry_run_label)
-    )
-    logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+    ping_url = f"{hook_config['ping_url']}/{MONITOR_STATE_TO_CRONITOR[state]}"
+
+    logger.info(f'{config_filename}: Pinging Cronitor {state.name.lower()}{dry_run_label}')
+    logger.debug(f'{config_filename}: Using Cronitor ping URL {ping_url}')
 
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)

borgmatic/hooks/dispatch.py

@@ -9,6 +9,7 @@ from borgmatic.hooks import (
     ntfy,
     pagerduty,
     postgresql,
+    sqlite,
 )
 
 logger = logging.getLogger(__name__)
@@ -22,6 +23,7 @@ HOOK_NAME_TO_MODULE = {
     'ntfy': ntfy,
     'pagerduty': pagerduty,
     'postgresql_databases': postgresql,
+    'sqlite_databases': sqlite,
 }
 
 
@@ -41,9 +43,9 @@ def call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs):
     try:
         module = HOOK_NAME_TO_MODULE[hook_name]
     except KeyError:
-        raise ValueError('Unknown hook name: {}'.format(hook_name))
+        raise ValueError(f'Unknown hook name: {hook_name}')
 
-    logger.debug('{}: Calling {} hook function {}'.format(log_prefix, hook_name, function_name))
+    logger.debug(f'{log_prefix}: Calling {hook_name} hook function {function_name}')
     return getattr(module, function_name)(config, log_prefix, *args, **kwargs)
 
 

borgmatic/hooks/dump.py

@@ -6,7 +6,12 @@ from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
 
 logger = logging.getLogger(__name__)
 
-DATABASE_HOOK_NAMES = ('postgresql_databases', 'mysql_databases', 'mongodb_databases')
+DATABASE_HOOK_NAMES = (
+    'postgresql_databases',
+    'mysql_databases',
+    'mongodb_databases',
+    'sqlite_databases',
+)
 
 
 def make_database_dump_path(borgmatic_source_directory, database_hook_name):
@@ -28,7 +33,7 @@ def make_database_dump_filename(dump_path, name, hostname=None):
     Raise ValueError if the database name is invalid.
     '''
     if os.path.sep in name:
-        raise ValueError('Invalid database name {}'.format(name))
+        raise ValueError(f'Invalid database name {name}')
 
     return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
 
@@ -55,9 +60,7 @@ def remove_database_dumps(dump_path, database_type_name, log_prefix, dry_run):
     '''
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
-    logger.debug(
-        '{}: Removing {} database dumps{}'.format(log_prefix, database_type_name, dry_run_label)
-    )
+    logger.debug(f'{log_prefix}: Removing {database_type_name} database dumps{dry_run_label}')
 
     expanded_path = os.path.expanduser(dump_path)
 
@@ -73,4 +76,4 @@ def convert_glob_patterns_to_borg_patterns(patterns):
     Convert a sequence of shell glob patterns like "/etc/*" to the corresponding Borg archive
     patterns like "sh:etc/*".
     '''
-    return ['sh:{}'.format(pattern.lstrip(os.path.sep)) for pattern in patterns]
+    return [f'sh:{pattern.lstrip(os.path.sep)}' for pattern in patterns]

borgmatic/hooks/healthchecks.py

@@ -10,6 +10,7 @@ MONITOR_STATE_TO_HEALTHCHECKS = {
     monitor.State.START: 'start',
     monitor.State.FINISH: None,  # Healthchecks doesn't append to the URL for the finished state.
     monitor.State.FAIL: 'fail',
+    monitor.State.LOG: 'log',
 }
 
 PAYLOAD_TRUNCATION_INDICATOR = '...\n'
@@ -98,7 +99,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
     ping_url = (
         hook_config['ping_url']
         if hook_config['ping_url'].startswith('http')
-        else 'https://hc-ping.com/{}'.format(hook_config['ping_url'])
+        else f"https://hc-ping.com/{hook_config['ping_url']}"
     )
     dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
 
@@ -110,14 +111,12 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
 
     healthchecks_state = MONITOR_STATE_TO_HEALTHCHECKS.get(state)
     if healthchecks_state:
-        ping_url = '{}/{}'.format(ping_url, healthchecks_state)
+        ping_url = f'{ping_url}/{healthchecks_state}'
 
-    logger.info(
-        '{}: Pinging Healthchecks {}{}'.format(config_filename, state.name.lower(), dry_run_label)
-    )
-    logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
+    logger.info(f'{config_filename}: Pinging Healthchecks {state.name.lower()}{dry_run_label}')
+    logger.debug(f'{config_filename}: Using Healthchecks ping URL {ping_url}')
 
-    if state in (monitor.State.FINISH, monitor.State.FAIL):
+    if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG):
         payload = format_buffered_logs_for_payload()
     else:
         payload = ''

borgmatic/hooks/mongodb.py

@@ -27,7 +27,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
 
-    logger.info('{}: Dumping MongoDB databases{}'.format(log_prefix, dry_run_label))
+    logger.info(f'{log_prefix}: Dumping MongoDB databases{dry_run_label}')
 
     processes = []
     for database in databases:
@@ -38,9 +38,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
         dump_format = database.get('format', 'archive')
 
         logger.debug(
-            '{}: Dumping MongoDB database {} to {}{}'.format(
-                log_prefix, name, dump_filename, dry_run_label
-            )
+            f'{log_prefix}: Dumping MongoDB database {name} to {dump_filename}{dry_run_label}',
         )
         if dry_run:
             continue
@@ -126,9 +124,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     )
     restore_command = build_restore_command(extract_process, database, dump_filename)
 
-    logger.debug(
-        '{}: Restoring MongoDB database {}{}'.format(log_prefix, database['name'], dry_run_label)
-    )
+    logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}")
     if dry_run:
         return
 

borgmatic/hooks/monitor.py

@@ -7,3 +7,4 @@ class State(Enum):
     START = 1
     FINISH = 2
     FAIL = 3
+    LOG = 4

borgmatic/hooks/mysql.py

@@ -88,9 +88,7 @@ def execute_dump_command(
         + (('--user', database['username']) if 'username' in database else ())
         + ('--databases',)
         + database_names
-        # Use shell redirection rather than execute_command(output_file=open(...)) to prevent
-        # the open() call on a named pipe from hanging the main borgmatic process.
-        + ('>', dump_filename)
+        + ('--result-file', dump_filename)
     )
 
     logger.debug(
@@ -102,7 +100,7 @@ def execute_dump_command(
     dump.create_named_pipe_for_dump(dump_filename)
 
     return execute_command(
-        dump_command, shell=True, extra_environment=extra_environment, run_to_completion=False,
+        dump_command, extra_environment=extra_environment, run_to_completion=False,
     )
 
 
@@ -119,7 +117,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
 
-    logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
+    logger.info(f'{log_prefix}: Dumping MySQL databases{dry_run_label}')
 
     for database in databases:
         dump_path = make_dump_path(location_config)
@@ -209,9 +207,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     )
     extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
 
-    logger.debug(
-        '{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
-    )
+    logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}")
     if dry_run:
         return
 

borgmatic/hooks/ntfy.py

@@ -2,16 +2,8 @@ import logging
 
 import requests
 
-from borgmatic.hooks import monitor
-
 logger = logging.getLogger(__name__)
 
-MONITOR_STATE_TO_NTFY = {
-    monitor.State.START: None,
-    monitor.State.FINISH: None,
-    monitor.State.FAIL: None,
-}
-
 
 def initialize_monitor(
     ping_url, config_filename, monitoring_log_level, dry_run

borgmatic/hooks/pagerduty.py

@@ -29,14 +29,12 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
     '''
     if state != monitor.State.FAIL:
         logger.debug(
-            '{}: Ignoring unsupported monitoring {} in PagerDuty hook'.format(
-                config_filename, state.name.lower()
-            )
+            f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in PagerDuty hook',
         )
         return
 
     dry_run_label = ' (dry run; not actually sending)' if dry_run else ''
-    logger.info('{}: Sending failure event to PagerDuty {}'.format(config_filename, dry_run_label))
+    logger.info(f'{config_filename}: Sending failure event to PagerDuty {dry_run_label}')
 
     if dry_run:
         return
@@ -50,7 +48,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
             'routing_key': hook_config['integration_key'],
             'event_action': 'trigger',
             'payload': {
-                'summary': 'backup failed on {}'.format(hostname),
+                'summary': f'backup failed on {hostname}',
                 'severity': 'error',
                 'source': hostname,
                 'timestamp': local_timestamp,
@@ -65,7 +63,7 @@ def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_
             },
         }
     )
-    logger.debug('{}: Using PagerDuty payload: {}'.format(config_filename, payload))
+    logger.debug(f'{config_filename}: Using PagerDuty payload: {payload}')
 
     logging.getLogger('urllib3').setLevel(logging.ERROR)
     try:

borgmatic/hooks/postgresql.py

@@ -93,7 +93,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
 
-    logger.info('{}: Dumping PostgreSQL databases{}'.format(log_prefix, dry_run_label))
+    logger.info(f'{log_prefix}: Dumping PostgreSQL databases{dry_run_label}')
 
     for database in databases:
         extra_environment = make_extra_environment(database)
@@ -228,9 +228,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     )
     extra_environment = make_extra_environment(database)
 
-    logger.debug(
-        '{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
-    )
+    logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")
     if dry_run:
         return
 

borgmatic/hooks/sqlite.py

@@ -0,0 +1,125 @@
+import logging
+import os
+
+from borgmatic.execute import execute_command, execute_command_with_processes
+from borgmatic.hooks import dump
+
+logger = logging.getLogger(__name__)
+
+
+def make_dump_path(location_config):  # pragma: no cover
+    '''
+    Make the dump path from the given location configuration and the name of this hook.
+    '''
+    return dump.make_database_dump_path(
+        location_config.get('borgmatic_source_directory'), 'sqlite_databases'
+    )
+
+
+def dump_databases(databases, log_prefix, location_config, dry_run):
+    '''
+    Dump the given SQLite3 databases to a file. The databases are supplied as a sequence of
+    configuration dicts, as per the configuration schema. Use the given log prefix in any log
+    entries. Use the given location configuration dict to construct the destination path. If this
+    is a dry run, then don't actually dump anything.
+    '''
+    dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+    processes = []
+
+    logger.info(f'{log_prefix}: Dumping SQLite databases{dry_run_label}')
+
+    for database in databases:
+        database_path = database['path']
+
+        if database['name'] == 'all':
+            logger.warning('The "all" database name has no meaning for SQLite3 databases')
+        if not os.path.exists(database_path):
+            logger.warning(
+                f'{log_prefix}: No SQLite database at {database_path}; An empty database will be created and dumped'
+            )
+
+        dump_path = make_dump_path(location_config)
+        dump_filename = dump.make_database_dump_filename(dump_path, database['name'])
+        if os.path.exists(dump_filename):
+            logger.warning(
+                f'{log_prefix}: Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}'
+            )
+            continue
+
+        command = (
+            'sqlite3',
+            database_path,
+            '.dump',
+            '>',
+            dump_filename,
+        )
+        logger.debug(
+            f'{log_prefix}: Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}'
+        )
+        if dry_run:
+            continue
+
+        dump.create_parent_directory_for_dump(dump_filename)
+        processes.append(execute_command(command, shell=True, run_to_completion=False))
+
+    return processes
+
+
+def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
+    '''
+    Remove the given SQLite3 database dumps from the filesystem. The databases are supplied as a
+    sequence of configuration dicts, as per the configuration schema. Use the given log prefix in
+    any log entries. Use the given location configuration dict to construct the destination path.
+    If this is a dry run, then don't actually remove anything.
+    '''
+    dump.remove_database_dumps(make_dump_path(location_config), 'SQLite', log_prefix, dry_run)
+
+
+def make_database_dump_pattern(
+    databases, log_prefix, location_config, name=None
+):  # pragma: no cover
+    '''
+    Make a pattern that matches the given SQLite3 databases. The databases are supplied as a
+    sequence of configuration dicts, as per the configuration schema.
+    '''
+    return dump.make_database_dump_filename(make_dump_path(location_config), name)
+
+
+def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+    '''
+    Restore the given SQLite3 database from an extract stream. The database is supplied as a
+    one-element sequence containing a dict describing the database, as per the configuration schema.
+    Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
+    anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
+    output to consume.
+    '''
+    dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
+
+    if len(database_config) != 1:
+        raise ValueError('The database configuration value is invalid')
+
+    database_path = database_config[0]['path']
+
+    logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
+    if dry_run:
+        return
+
+    try:
+        os.remove(database_path)
+        logger.warning(f'{log_prefix}: Removed existing SQLite database at {database_path}')
+    except FileNotFoundError:  # pragma: no cover
+        pass
+
+    restore_command = (
+        'sqlite3',
+        database_path,
+    )
+
+    # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
+    # if the restore paths don't exist in the archive.
+    execute_command_with_processes(
+        restore_command,
+        [extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    )

borgmatic/logger.py

@@ -68,7 +68,7 @@ class Multi_stream_handler(logging.Handler):
 
     def emit(self, record):
         '''
-        Dispatch the log record to the approriate stream handler for the record's log level.
+        Dispatch the log record to the appropriate stream handler for the record's log level.
         '''
         self.log_level_to_handler[record.levelno].emit(record)
 
@@ -108,7 +108,7 @@ def color_text(color, message):
     if not color:
         return message
 
-    return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL)
+    return f'{color}{message}{colorama.Style.RESET_ALL}'
 
 
 def add_logging_level(level_name, level_number):
@@ -156,6 +156,7 @@ def configure_logging(
     log_file_log_level=None,
     monitoring_log_level=None,
     log_file=None,
+    log_file_format=None,
 ):
     '''
     Configure logging to go to both the console and (syslog or log file). Use the given log levels,
@@ -200,12 +201,18 @@ def configure_logging(
 
     if syslog_path and not interactive_console():
         syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
-        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
+        syslog_handler.setFormatter(
+            logging.Formatter('borgmatic: {levelname} {message}', style='{')  # noqa: FS003
+        )
         syslog_handler.setLevel(syslog_log_level)
         handlers = (console_handler, syslog_handler)
     elif log_file:
         file_handler = logging.handlers.WatchedFileHandler(log_file)
-        file_handler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s'))
+        file_handler.setFormatter(
+            logging.Formatter(
+                log_file_format or '[{asctime}] {levelname}: {message}', style='{'  # noqa: FS003
+            )
+        )
         file_handler.setLevel(log_file_log_level)
         handlers = (console_handler, file_handler)
     else:

docs/Dockerfile

@@ -1,14 +1,14 @@
-FROM alpine:3.17.1 as borgmatic
+FROM docker.io/alpine:3.17.1 as borgmatic
 
 COPY . /app
 RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
-    && for action in rcreate transfer prune compact create check extract export-tar mount umount restore rlist list rinfo info break-lock borg; do \
+    && for action in rcreate transfer create prune compact check extract export-tar mount umount restore rlist list rinfo info break-lock borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:19.5.0-alpine as html
+FROM docker.io/node:19.5.0-alpine as html
 
 ARG ENVIRONMENT=production
 
@@ -18,6 +18,7 @@ RUN npm install @11ty/eleventy \
     @11ty/eleventy-plugin-syntaxhighlight \
     @11ty/eleventy-plugin-inclusive-language \
     @11ty/eleventy-navigation \
+    eleventy-plugin-code-clipboard \
     markdown-it \
     markdown-it-anchor \
     markdown-it-replace-link
@@ -27,7 +28,7 @@ COPY . /source
 RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.22.1-alpine
+FROM docker.io/nginx:1.22.1-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/_includes/components/lists.css

@@ -63,11 +63,6 @@
 		top: -2px;
 		bottom: 2px;
 	}
-	@media (prefers-color-scheme: dark) {
-		.inlinelist .inlinelist-item code:before {
-			border-left-color: rgba(0,0,0,.8);
-		}
-	}
 }
 a.buzzword {
 	text-decoration: underline;
@@ -91,26 +86,9 @@ a.buzzword {
 .buzzword {
 	background-color: #f7f7f7;
 }
-@media (prefers-color-scheme: dark) {
-	.buzzword-list li,
-	.buzzword {
-		background-color: #080808;
-	}
-}
 .inlinelist .inlinelist-item {
 	background-color: #e9e9e9;
 }
-@media (prefers-color-scheme: dark) {
-	.inlinelist .inlinelist-item {
-		background-color: #000;
-	}
-	.inlinelist .inlinelist-item a {
-		color: #fff;
-	}
-	.inlinelist .inlinelist-item code {
-		color: inherit;
-	}
-}
 .inlinelist .inlinelist-item:hover,
 .inlinelist .inlinelist-item:focus,
 .buzzword-list li:hover,
@@ -217,12 +195,6 @@ main p a.buzzword {
 	height: 1.75em;
 	font-weight: 600;
 }
-@media (prefers-color-scheme: dark) {
-	.numberflag {
-		background-color: #00bcd4;
-		color: #222;
-	}
-}
 h1 .numberflag,
 h2 .numberflag,
 h3 .numberflag,
@@ -244,11 +216,6 @@ h2 .numberflag:after {
 	background-color: #fff;
 	width: calc(100% + 0.4em); /* 16px /40 */
 }
-@media (prefers-color-scheme: dark) {
-	h2 .numberflag:after {
-		background-color: #222;
-	}
-}
 
 /* Super featured list on home page */
 .list-superfeatured .avatar {

docs/_includes/components/minilink.css

@@ -12,16 +12,6 @@
 	line-height: 1.285714285714; /* 18px /14 */
 	font-family: system-ui, -apple-system, sans-serif;
 }
-@media (prefers-color-scheme: dark) {
-  .minilink {
-    background-color: #222;
-    /*
-      !important to override .elv-callout a
-      see _includes/components/callout.css
-     */
-    color: #fff !important;
-  }
-}
 table .minilink {
 	margin-top: 6px;
 }
@@ -32,12 +22,6 @@ table .minilink {
 .minilink[href]:focus {
 	background-color: #bbb;
 }
-@media (prefers-color-scheme: dark) {
-  .minilink[href]:hover,
-  .minilink[href]:focus {
-    background-color: #444;
-  }
-}
 pre + .minilink {
 	color: #fff;
 	border-radius: 0 0 0.2857142857143em 0.2857142857143em; /* 4px /14 */
@@ -74,11 +58,6 @@ h4 .minilink {
 	text-transform: none;
 	box-shadow: 0 0 0 1px rgba(0,0,0,0.3);
 }
-@media (prefers-color-scheme: dark) {
-  .minilink-addedin {
-    box-shadow: 0 0 0 1px rgba(255,255,255,0.3);
-  }
-}
 .minilink-addedin:not(:first-child) {
 	margin-left: .5em;
 }

docs/_includes/components/toc.css

@@ -79,22 +79,11 @@
 	border-bottom: 1px solid #ddd;
 	margin-bottom: 0.25em; /* 4px /16 */
 }
-@media (prefers-color-scheme: dark) {
-  .elv-toc-list > li > a {
-    color: #fff;
-    border-color: #444;
-  }
-}
 
 /* Active links */
 .elv-toc-list li.elv-toc-active > a {
 	background-color: #dff7ff;
 }
-@media (prefers-color-scheme: dark) {
-  .elv-toc-list li.elv-toc-active > a {
-    background-color: #353535;
-  }
-}
 .elv-toc-list ul .elv-toc-active > a:after {
 	content: "";
 }
@@ -105,7 +94,7 @@
 	display: block;
 }
 
-/* Footer catgory navigation */
+/* Footer category navigation */
 .elv-cat-list-active {
 	font-weight: 600;
 }

docs/_includes/index.css

@@ -285,11 +285,6 @@ footer.elv-layout {
 .elv-hero {
 	background-color: #222;
 }
-@media (prefers-color-scheme: dark) {
-	.elv-hero {
-		background-color: #292929;
-	}
-}
 .elv-hero img,
 .elv-hero svg {
 	width: 42.95774646vh;
@@ -538,3 +533,18 @@ main .elv-toc + h1 .direct-link {
 .header-anchor:hover::after {
     content: " 🔗";
 }
+
+.mdi {
+    display: inline-block;
+    width: 1em;
+    height: 1em;
+    background-color: currentColor;
+    -webkit-mask: no-repeat center / 100%;
+    mask: no-repeat center / 100%;
+    -webkit-mask-image: var(--svg);
+    mask-image: var(--svg);
+}
+
+.mdi.mdi-content-copy {
+    --svg: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' width='24' height='24'%3E%3Cpath fill='black' d='M19 21H8V7h11m0-2H8a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h11a2 2 0 0 0 2-2V7a2 2 0 0 0-2-2m-3-4H4a2 2 0 0 0-2 2v14h2V3h12V1Z'/%3E%3C/svg%3E");
+}

docs/_includes/layouts/base.njk

@@ -3,6 +3,7 @@
 	<head>
 		<meta charset="utf-8">
 		<meta name="viewport" content="width=device-width, initial-scale=1.0">
+		<link rel="icon" href="docs/static/borgmatic.png" type="image/x-icon">
 		<title>{{ subtitle + ' - ' if subtitle}}{{ title }}</title>
 {%- set css %}
 {% include 'index.css' %}
@@ -22,6 +23,6 @@
 	<body>
 
 		{{ content | safe }}
-
+		{% initClipboardJS %}
 	</body>
 </html>

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -66,6 +66,9 @@ variables you can use here:
 
  * `configuration_filename`: borgmatic configuration filename in which the
    hook was defined
+ * `log_file`
+   <span class="minilink minilink-addedin">New in version 1.7.12</span>:
+   path of the borgmatic log file, only set when the `--log-file` flag is used
  * `repository`: path of the current repository as configured in the current
    borgmatic configuration file
 

docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md

@@ -49,9 +49,12 @@ location:
         - /home
 
     repositories:
-        - /mnt/removable/backup.borg
+        - path: /mnt/removable/backup.borg
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.10</span> Omit
+the `path:` portion of the `repositories` list.
+
 Then, write a `before_backup` hook in that same configuration file that uses
 the external `findmnt` utility to see whether the drive is mounted before
 proceeding.
@@ -79,13 +82,16 @@ location:
         - /home
 
     repositories:
-        - ssh://me@buddys-server.org/./backup.borg
+        - path: ssh://me@buddys-server.org/./backup.borg
 
 hooks:
     before_backup:
       - ping -q -c 1 buddys-server.org > /dev/null || exit 75
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.10</span> Omit
+the `path:` portion of the `repositories` list.
+
 Or to only run backups if the battery level is high enough:
 
 ```yaml
@@ -110,8 +116,8 @@ There are some caveats you should be aware of with this feature.
  * You'll generally want to put a soft failure command in the `before_backup`
    hook, so as to gate whether the backup action occurs. While a soft failure is
    also supported in the `after_backup` hook, returning a soft failure there
-   won't prevent any actions from occuring, because they've already occurred!
-   Similiarly, you can return a soft failure from an `on_error` hook, but at
+   won't prevent any actions from occurring, because they've already occurred!
+   Similarly, you can return a soft failure from an `on_error` hook, but at
    that point it's too late to prevent the error.
  * Returning a soft failure does prevent further commands in the same hook from
    executing. So, like a standard error, it is an "early out". Unlike a standard

docs/how-to/backup-your-databases.md

@@ -15,8 +15,7 @@ consistent snapshot that is more suited for backups.
 
 Fortunately, borgmatic includes built-in support for creating database dumps
 prior to running backups. For example, here is everything you need to dump and
-backup a couple of local PostgreSQL databases, a MySQL/MariaDB database, and a
-MongoDB database:
+backup a couple of local PostgreSQL databases and a MySQL/MariaDB database.
 
 ```yaml
 hooks:
@@ -25,10 +24,27 @@ hooks:
         - name: orders
     mysql_databases:
         - name: posts
+```
+
+<span class="minilink minilink-addedin">New in version 1.5.22</span> You can
+also dump MongoDB databases. For example:
+
+```yaml
+hooks:
     mongodb_databases:
         - name: messages
 ```
 
+<span class="minilink minilink-addedin">New in version 1.7.9</span>
+Additionally, you can dump SQLite databases. For example:
+
+```yaml
+hooks:
+    sqlite_databases:
+        - name: mydb
+          path: /var/lib/sqlite3/mydb.sqlite
+```
+
 As part of each backup, borgmatic streams a database dump for each configured
 database directly to Borg, so it's included in the backup without consuming
 additional disk space. (The exceptions are the PostgreSQL/MongoDB "directory"
@@ -74,6 +90,9 @@ hooks:
           password: trustsome1
           authentication_database: mongousers
           options: "--ssl"
+    sqlite_databases:
+        - name: mydb
+          path: /var/lib/sqlite3/mydb.sqlite
 ```
 
 See your [borgmatic configuration
@@ -99,6 +118,8 @@ hooks:
 Note that you may need to use a `username` of the `postgres` superuser for
 this to work with PostgreSQL.
 
+The SQLite hook in particular does not consider "all" a special database name.
+
 <span class="minilink minilink-addedin">New in version 1.7.6</span> With
 PostgreSQL and MySQL, you can optionally dump "all" databases to separate
 files instead of one combined dump file, allowing more convenient restores of
@@ -115,6 +136,53 @@ hooks:
           format: sql
 ```
 
+### Containers
+
+If your database is running within a Docker container and borgmatic is too, no
+problem—simply configure borgmatic to connect to the container's name on its
+exposed port. For instance:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: users
+          hostname: your-database-container-name
+          port: 5433
+          username: postgres
+          password: trustsome1
+```
+
+But what if borgmatic is running on the host? You can still connect to a
+database container if its ports are properly exposed to the host. For
+instance, when running the database container with Docker, you can specify
+`--publish 127.0.0.1:5433:5432` so that it exposes the container's port 5432
+to port 5433 on the host (only reachable on localhost, in this case). Or the
+same thing with Docker Compose:
+
+```yaml
+services:
+   your-database-container-name:
+       image: postgres
+       ports:
+           - 127.0.0.1:5433:5432
+```
+
+And then you can connect to the database from borgmatic running on the host:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: users
+          hostname: 127.0.0.1
+          port: 5433
+          username: postgres
+          password: trustsome1
+```
+
+Of course, alter the ports in these examples to suit your particular database
+system.
+
+
 ### No source directories
 
 <span class="minilink minilink-addedin">New in version 1.7.1</span> If you
@@ -133,7 +201,6 @@ hooks:
 ```
 
 
-
 ### External passwords
 
 If you don't want to keep your database passwords in your borgmatic
@@ -154,11 +221,11 @@ bring back any missing configuration files in order to restore a database.
 
 ## Supported databases
 
-As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, and MongoDB databases
-directly. But see below about general-purpose preparation and cleanup hooks as
-a work-around with other database systems. Also, please [file a
-ticket](https://torsion.org/borgmatic/#issues) for additional database systems
-that you'd like supported.
+As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, MongoDB, and SQLite
+databases directly. But see below about general-purpose preparation and
+cleanup hooks as a work-around with other database systems. Also, please [file
+a ticket](https://torsion.org/borgmatic/#issues) for additional database
+systems that you'd like supported.
 
 
 ## Database restoration
@@ -210,7 +277,8 @@ If you have a single repository in your borgmatic configuration file(s), no
 problem: the `restore` action figures out which repository to use.
 
 But if you have multiple repositories configured, then you'll need to specify
-the repository path containing the archive to restore. Here's an example:
+the repository to use via the `--repository` flag. This can be done either
+with the repository's path or its label as configured in your borgmatic configuration file.
 
 ```bash
 borgmatic restore --repository repo.borg --archive host-2023-...
@@ -295,7 +363,10 @@ user and you're extracting to `/tmp`, then the dump will be in
 `/tmp/root/.borgmatic`.
 
 After extraction, you can manually restore the dump file using native database
-commands like `pg_restore`, `mysql`, `mongorestore` or similar.
+commands like `pg_restore`, `mysql`, `mongorestore`, `sqlite`, or similar.
+
+Also see the documentation on [listing database
+dumps](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#listing-database-dumps).
 
 
 ## Preparation and cleanup hooks
@@ -310,6 +381,23 @@ dumps with any database system.
 
 ## Troubleshooting
 
+### PostgreSQL/MySQL authentication errors
+
+With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors
+when borgmatic tries to connect to your database, a natural reaction is to
+increase your borgmatic verbosity with `--verbosity 2` and go looking in the
+logs. You'll notice however that your database password does not show up in
+the logs. This is likely not the cause of the authentication problem unless
+you mistyped your password, however; borgmatic passes your password to the
+database via an environment variable that does not appear in the logs.
+
+The cause of an authentication error is often on the database side—in the
+configuration of which users are allowed to connect and how they are
+authenticated. For instance, with PostgreSQL, check your
+[pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html)
+file for that configuration.
+
+
 ### MySQL table lock errors
 
 If you encounter table lock errors during a database dump with MySQL/MariaDB,

docs/how-to/deal-with-very-large-backups.md

@@ -9,37 +9,47 @@ eleventyNavigation:
 
 Borg itself is great for efficiently de-duplicating data across successive
 backup archives, even when dealing with very large repositories. But you may
-find that while borgmatic's default mode of `prune`, `compact`, `create`, and
-`check` works well on small repositories, it's not so great on larger ones.
-That's because running the default pruning, compact, and consistency checks
-take a long time on large repositories.
+find that while borgmatic's default actions of `create`, `prune`, `compact`,
+and `check` works well on small repositories, it's not so great on larger
+ones. That's because running the default pruning, compact, and consistency
+checks take a long time on large repositories.
+
+<span class="minilink minilink-addedin">Prior to version 1.7.9</span> The
+default action ordering was `prune`, `compact`, `create`, and `check`.
 
 ### A la carte actions
 
-If you find yourself in this situation, you have some options. First, you can
-run borgmatic's `prune`, `compact`, `create`, or `check` actions separately.
-For instance, the following optional actions are available:
+If you find yourself wanting to customize the actions, you have some options.
+First, you can run borgmatic's `prune`, `compact`, `create`, or `check`
+actions separately. For instance, the following optional actions are
+available (among others):
 
 ```bash
+borgmatic create
 borgmatic prune
 borgmatic compact
-borgmatic create
 borgmatic check
 ```
 
-You can run with only one of these actions provided, or you can mix and match
-any number of them in a single borgmatic run. This supports approaches like
-skipping certain actions while running others. For instance, this skips
-`prune` and `compact` and only runs `create` and `check`:
+You can run borgmatic with only one of these actions provided, or you can mix
+and match any number of them in a single borgmatic run. This supports
+approaches like skipping certain actions while running others. For instance,
+this skips `prune` and `compact` and only runs `create` and `check`:
 
 ```bash
 borgmatic create check
 ```
 
-Or, you can make backups with `create` on a frequent schedule (e.g. with
-`borgmatic create` called from one cron job), while only running expensive
-consistency checks with `check` on a much less frequent basis (e.g. with
-`borgmatic check` called from a separate cron job).
+<span class="minilink minilink-addedin">New in version 1.7.9</span> borgmatic
+now respects your specified command-line action order, running actions in the
+order you specify. In previous versions, borgmatic ran your specified actions
+in a fixed ordering regardless of the order they appeared on the command-line.
+
+But instead of running actions together, another option is to run backups with
+`create` on a frequent schedule (e.g. with `borgmatic create` called from one
+cron job), while only running expensive consistency checks with `check` on a
+much less frequent basis (e.g. with `borgmatic check` called from a separate
+cron job).
 
 
 ### Consistency check configuration
@@ -47,8 +57,8 @@ consistency checks with `check` on a much less frequent basis (e.g. with
 Another option is to customize your consistency checks. By default, if you
 omit consistency checks from configuration, borgmatic runs full-repository
 checks (`repository`) and per-archive checks (`archives`) within each
-repository, no more than once a month. This is equivalent to what `borg check`
-does if run without options.
+repository. (Although see below about check frequency.) This is equivalent to
+what `borg check` does if run without options.
 
 But if you find that archive checks are too slow, for example, you can
 configure borgmatic to run repository checks only. Configure this in the
@@ -60,8 +70,9 @@ consistency:
         - name: repository
 ```
 
-<span class="minilink minilink-addedin">Prior to version 1.6.2</span> `checks`
-was a plain list of strings without the `name:` part. For example:
+<span class="minilink minilink-addedin">Prior to version 1.6.2</span> The
+`checks` option was a plain list of strings without the `name:` part, and
+borgmatic ran each configured check every time checks were run. For example:
 
 ```yaml
 consistency:
@@ -102,8 +113,13 @@ consistency:
 This tells borgmatic to run the `repository` consistency check at most once
 every two weeks for a given repository and the `archives` check at most once a
 month. The `frequency` value is a number followed by a unit of time, e.g. "3
-days", "1 week", "2 months", etc. The `frequency` defaults to `always`, which
-means run this check every time checks run.
+days", "1 week", "2 months", etc.
+
+The `frequency` defaults to `always` for a check configured without a
+`frequency`, which means run this check every time checks run. But if you omit
+consistency checks from configuration entirely, borgmatic runs full-repository
+checks (`repository`) and per-archive checks (`archives`) within each
+repository, at most once a month.
 
 Unlike a real scheduler like cron, borgmatic only makes a best effort to run
 checks on the configured frequency. It compares that frequency with how long

docs/how-to/develop-on-borgmatic.md

@@ -25,8 +25,8 @@ so that you can run borgmatic commands while you're hacking on them to
 make sure your changes work.
 
 ```bash
-cd borgmatic/
-pip3 install --editable --user .
+cd borgmatic
+pip3 install --user --editable .
 ```
 
 Note that this will typically install the borgmatic commands into
@@ -51,7 +51,6 @@ pip3 install --user tox
 Finally, to actually run tests, run:
 
 ```bash
-cd borgmatic
 tox
 ```
 
@@ -74,6 +73,15 @@ can ask isort to order your imports for you:
 tox -e isort
 ```
 
+Similarly, if you get errors about spelling mistakes in source code, you can
+ask [codespell](https://github.com/codespell-project/codespell) to correct
+them:
+
+```bash
+tox -e codespell
+```
+
+
 ### End-to-end tests
 
 borgmatic additionally includes some end-to-end tests that integration test
@@ -87,12 +95,36 @@ If you would like to run the full test suite, first install Docker and [Docker
 Compose](https://docs.docker.com/compose/install/). Then run:
 
 ```bash
-scripts/run-full-dev-tests
+scripts/run-end-to-end-dev-tests
 ```
 
 Note that this scripts assumes you have permission to run Docker. If you
 don't, then you may need to run with `sudo`.
 
+
+#### Podman
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span>
+borgmatic's end-to-end tests optionally support using
+[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md)
+[Podman](https://podman.io/) instead of Docker.
+
+Setting up Podman is outside the scope of this documentation, but here are
+some key points to double-check:
+
+ * Install Podman along with `podman-docker` and your desired networking
+   support.
+ * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the
+   non-root user who will run tests.
+ * Create a non-root Podman socket for that user:
+   ```bash
+   systemctl --user enable --now podman.socket
+   ```
+
+Then you'll be able to run end-to-end tests as per normal, and the test script
+will automatically use your non-root Podman socket instead of a Docker socket.
+
+
 ## Code style
 
 Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply
@@ -101,10 +133,10 @@ the following deviations from it:
  * For strings, prefer single quotes over double quotes.
  * Limit all lines to a maximum of 100 characters.
  * Use trailing commas within multiline values or argument lists.
- * For multiline constructs, put opening and closing delimeters on lines
+ * For multiline constructs, put opening and closing delimiters on lines
    separate from their contents.
  * Within multiline constructs, use standard four-space indentation. Don't align
-   indentation with an opening delimeter.
+   indentation with an opening delimiter.
 
 borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
 formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and
@@ -141,3 +173,15 @@ http://localhost:8080 to view the documentation with your changes.
 To close the documentation server, ctrl-C the script. Note that it does not
 currently auto-reload, so you'll need to stop it and re-run it for any
 additional documentation changes to take effect.
+
+
+#### Podman
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span>
+borgmatic's developer build for documentation optionally supports using
+[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md)
+[Podman](https://podman.io/) instead of Docker.
+
+Setting up Podman is outside the scope of this documentation. But once you
+install `podman-docker`, then `scripts/dev-docs` should automatically use
+Podman instead of Docker.

docs/how-to/extract-a-backup.md

@@ -20,15 +20,15 @@ borgmatic rlist
 That should yield output looking something like:
 
 ```text
-host-2019-01-01T04:05:06.070809      Tue, 2019-01-01 04:05:06 [...]
-host-2019-01-02T04:06:07.080910      Wed, 2019-01-02 04:06:07 [...]
+host-2023-01-01T04:05:06.070809      Tue, 2023-01-01 04:05:06 [...]
+host-2023-01-02T04:06:07.080910      Wed, 2023-01-02 04:06:07 [...]
 ```
 
 Assuming that you want to extract the archive with the most up-to-date files
 and therefore the latest timestamp, run a command like:
 
 ```bash
-borgmatic extract --archive host-2019-01-02T04:06:07.080910
+borgmatic extract --archive host-2023-01-02T04:06:07.080910
 ```
 
 (No borgmatic `extract` action? Upgrade borgmatic!)
@@ -51,10 +51,11 @@ If you have a single repository in your borgmatic configuration file(s), no
 problem: the `extract` action figures out which repository to use.
 
 But if you have multiple repositories configured, then you'll need to specify
-the repository path containing the archive to extract. Here's an example:
+the repository to use via the `--repository` flag. This can be done either
+with the repository's path or its label as configured in your borgmatic configuration file.
 
 ```bash
-borgmatic extract --repository repo.borg --archive host-2019-...
+borgmatic extract --repository repo.borg --archive host-2023-...
 ```
 
 ## Extract particular files
@@ -74,6 +75,13 @@ run the `extract` command above, borgmatic will extract `/var/path/1` and
 `/var/path/2`.
 
 
+### Searching for files
+
+If you're not sure which archive contains the files you're looking for, you
+can [search across
+archives](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#searching-for-a-file).
+
+
 ## Extract to a particular destination
 
 By default, borgmatic extracts files into the current directory. To instead

docs/how-to/inspect-your-backups.md

@@ -91,6 +91,19 @@ example, to search only the last five archives:
 borgmatic list --find foo.txt --last 5
 ```
 
+## Listing database dumps
+
+If you have enabled borgmatic's [database
+hooks](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/), you
+can list backed up database dumps via borgmatic. For example:
+
+```bash 
+borgmatic list --archive latest --find .borgmatic/*_databases
+```
+
+This gives you a listing of all database dump files contained in the latest
+archive, complete with file sizes.
+
 
 ## Logging
 
@@ -98,7 +111,7 @@ By default, borgmatic logs to a local syslog-compatible daemon if one is
 present and borgmatic is running in a non-interactive console. Where those
 logs show up depends on your particular system. If you're using systemd, try
 running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or
-similiar.
+similar.
 
 You can customize the log level used for syslog logging with the
 `--syslog-verbosity` flag, and this is independent from the console logging
@@ -141,5 +154,39 @@ borgmatic --log-file /path/to/file.log
 
 Note that if you use the `--log-file` flag, you are responsible for rotating
 the log file so it doesn't grow too large, for example with
-[logrotate](https://wiki.archlinux.org/index.php/Logrotate). Also, there is a
-`--log-file-verbosity` flag to customize the log file's log level.
+[logrotate](https://wiki.archlinux.org/index.php/Logrotate).
+
+You can the `--log-file-verbosity` flag to customize the log file's log level:
+
+```bash
+borgmatic --log-file /path/to/file.log --log-file-verbosity 2
+```
+
+<span class="minilink minilink-addedin">New in version 1.7.11</span> Use the
+`--log-file-format` flag to override the default log message format. This
+format string can contain a series of named placeholders wrapped in curly
+brackets. For instance, the default log format is: `[{asctime}] {levelname}:
+{message}`. This means each log message is recorded as the log time (in square
+brackets), a logging level name, a colon, and the actual log message.
+
+So if you just want each log message to get logged *without* a timestamp or a
+logging level name:
+
+```bash
+borgmatic --log-file /path/to/file.log --log-file-format "{message}"
+```
+
+Here is a list of available placeholders:
+
+ * `{asctime}`: time the log message was created
+ * `{levelname}`: level of the log message (`INFO`, `DEBUG`, etc.)
+ * `{lineno}`: line number in the source file where the log message originated
+ * `{message}`: actual log message
+ * `{pathname}`: path of the source file where the log message originated
+
+See the [Python logging
+documentation](https://docs.python.org/3/library/logging.html#logrecord-attributes)
+for additional placeholders.
+
+Note that this `--log-file-format` flg only applies to the specified
+`--log-file` and not to syslog or other logging.

docs/how-to/make-backups-redundant.md

@@ -20,11 +20,13 @@ location:
 
     # Paths of local or remote repositories to backup to.
     repositories:
-        - ssh://1234@usw-s001.rsync.net/./backups.borg
-        - ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo
-        - /var/lib/backups/local.borg
+        - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo
+        - path: /var/lib/backups/local.borg
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.10</span> Omit
+the `path:` portion of the `repositories` list.
+
 When you run borgmatic with this configuration, it invokes Borg once for each
 configured repository in sequence. (So, not in parallel.) That means—in each
 repository—borgmatic creates a single new backup archive containing all of
@@ -32,9 +34,8 @@ your source directories.
 
 Here's a way of visualizing what borgmatic does with the above configuration:
 
-1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg`
-2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo`
-3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg`
+1. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo`
+2. Backup `/home` and `/etc` to `/var/lib/backups/local.borg`
 
 This gives you redundancy of your data across repositories and even
 potentially across providers.

docs/how-to/make-per-application-backups.md

@@ -54,6 +54,93 @@ choice](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot),
 each entry using borgmatic's `--config` flag instead of relying on
 `/etc/borgmatic.d`.
 
+
+## Archive naming
+
+If you've got multiple borgmatic configuration files, you might want to create
+archives with different naming schemes for each one. This is especially handy
+if each configuration file is backing up to the same Borg repository but you
+still want to be able to distinguish backup archives for one application from
+another.
+
+borgmatic supports this use case with an `archive_name_format` option. The
+idea is that you define a string format containing a number of [Borg
+placeholders](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-placeholders),
+and borgmatic uses that format to name any new archive it creates. For
+instance:
+
+```yaml
+storage:
+    ...
+    archive_name_format: home-directories-{now}
+```
+
+This means that when borgmatic creates an archive, its name will start with
+the string `home-directories-` and end with a timestamp for its creation time.
+If `archive_name_format` is unspecified, the default is
+`{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a
+timestamp in a particular format.
+
+<span class="minilink minilink-addedin">New in version 1.7.11</span> borgmatic
+uses the `archive_name_format` option to automatically limit which archives
+get used for actions operating on multiple archives. This prevents, for
+instance, duplicate archives from showing up in `rlist` or `info` results—even
+if the same repository appears in multiple borgmatic configuration files. To
+take advantage of this feature, simply use a different `archive_name_format`
+in each configuration file.
+
+Under the hood, borgmatic accomplishes this by substituting globs for certain
+ephemeral data placeholders in your `archive_name_format`—and using the result
+to filter archives when running supported actions.
+
+For instance, let's say that you have this in your configuration:
+
+```yaml
+storage:
+    ...
+    archive_name_format: {hostname}-user-data-{now}
+```
+
+borgmatic considers `{now}` an emphemeral data placeholder that will probably
+change per archive, while `{hostname}` won't. So it turns the example value
+into `{hostname}-user-data-*` and applies it to filter down the set of
+archives used for actions like `rlist`, `info`, `prune`, `check`, etc.
+
+The end result is that when borgmatic runs the actions for a particular
+application-specific configuration file, it only operates on the archives
+created for that application. Of course, this doesn't apply to actions like
+`compact` that operate on an entire repository.
+
+If this behavior isn't quite smart enough for your needs, you can use the
+`match_archives` option to override the pattern that borgmatic uses for
+filtering archives. For example:
+
+```yaml
+storage:
+    ...
+    archive_name_format: {hostname}-user-data-{now}
+    match_archives: sh:myhost-user-data-*        
+```
+
+For Borg 1.x, use a shell pattern for the `match_archives` value and see the
+[Borg patterns
+documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns)
+for more information. For Borg 2.x, see the [match archives
+documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/help.html#borg-help-match-archives).
+
+Some borgmatic command-line actions also have a `--match-archives` flag that
+overrides both the auto-matching behavior and the `match_archives`
+configuration option.
+
+<span class="minilink minilink-addedin">Prior to 1.7.11</span> The way to
+limit the archives used for the `prune` action was a `prefix` option in the
+`retention` section for matching against the start of archive names. And the
+option for limiting the archives used for the `check` action was a separate
+`prefix` in the `consistency` section. Both of these options are deprecated in
+favor of the auto-matching behavior (or `match_archives`/`--match-archives`)
+in newer versions of borgmatic.
+
+
 ## Configuration includes
 
 Once you have multiple different configuration files, you might want to share
@@ -189,6 +276,81 @@ include, the local file's option takes precedence.
 list values are appended together.
 
 
+### Shallow merge
+
+Even though deep merging is generally pretty handy for included files,
+sometimes you want specific sections in the local file to take precedence over
+included sections—without any merging occurring for them.
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span> That's
+where the `!retain` tag comes in. Whenever you're merging an included file
+into your configuration file, you can optionally add the `!retain` tag to
+particular local mappings or lists to retain the local values and ignore
+included values.
+
+For instance, start with this configuration file containing the `!retain` tag
+on the `retention` mapping:
+
+```yaml
+<<: !include /etc/borgmatic/common.yaml
+
+location:
+   repositories:
+     - repo.borg
+
+retention: !retain
+    keep_daily: 5
+```
+
+And `common.yaml` like this:
+
+```yaml
+location:
+   repositories:
+     - common.borg
+
+retention:
+    keep_hourly: 24
+    keep_daily: 7
+```
+
+Once this include gets merged in, the resulting configuration will have a
+`keep_daily` value of `5` and nothing else in the `retention` section. That's
+because the `!retain` tag says to retain the local version of `retention` and
+ignore any values coming in from the include. But because the `repositories`
+list doesn't have a `!retain` tag, it still gets merged together to contain
+both `common.borg` and `repo.borg`.
+
+The `!retain` tag can only be placed on mappings and lists, and it goes right
+after the name of the option (and its colon) on the same line. The effects of
+`!retain` are recursive, meaning that if you place a `!retain` tag on a
+top-level mapping, even deeply nested values within it will not be merged.
+
+Additionally, the `!retain` tag only works in a configuration file that also
+performs a merge include with `<<: !include`. It doesn't make sense within,
+for instance, an included configuration file itself (unless it in turn
+performs its own merge include). That's because `!retain` only applies to the
+file doing the include; it doesn't work in reverse or propagate through
+includes.
+
+
+## Debugging includes
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span> If you'd
+like to see what the loaded configuration looks like after includes get merged
+in, run `validate-borgmatic-config` on your configuration file:
+
+```bash
+sudo validate-borgmatic-config --show
+```
+
+You'll need to specify your configuration file with `--config` if it's not in
+a default location.
+
+This will output the merged configuration as borgmatic sees it, which can be
+helpful for understanding how your includes work in practice.
+
+
 ## Configuration overrides
 
 In more complex multi-application setups, you may want to override particular
@@ -255,3 +417,51 @@ Be sure to quote your overrides if they contain spaces or other characters
 that your shell may interpret.
 
 An alternate to command-line overrides is passing in your values via [environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).
+
+
+## Constant interpolation
+
+<span class="minilink minilink-addedin">New in version 1.7.10</span> Another
+tool is borgmatic's support for defining custom constants. This is similar to
+the [variable interpolation
+feature](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation)
+for command hooks, but the constants feature lets you substitute your own
+custom values into anywhere in the entire configuration file. (Constants don't
+work across includes or separate configuration files though.)
+
+Here's an example usage:
+
+```yaml
+constants:
+    user: foo
+    archive_prefix: bar
+
+location:
+    source_directories:
+        - /home/{user}/.config
+        - /home/{user}/.ssh
+    ...
+
+storage:
+    archive_name_format: '{archive_prefix}-{now}'
+```
+
+In this example, when borgmatic runs, all instances of `{user}` get replaced
+with `foo` and all instances of `{archive-prefix}` get replaced with `bar-`.
+(And in this particular example, `{now}` doesn't get replaced with anything,
+but gets passed directly to Borg.) After substitution, the logical result
+looks something like this:
+
+```yaml
+location:
+    source_directories:
+        - /home/foo/.config
+        - /home/foo/.ssh
+    ...
+
+storage:
+    archive_name_format: 'bar-{now}'
+```
+
+An alternate to constants is passing in your values via [environment
+variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).

docs/how-to/monitor-your-backups.md

@@ -83,7 +83,7 @@ tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/).
 
 ## Error hooks
 
-When an error occurs during a `prune`, `compact`, `create`, or `check` action,
+When an error occurs during a `create`, `prune`, `compact`, or `check` action,
 borgmatic can run configurable shell commands to fire off custom error
 notifications or take other actions, so you can get alerted as soon as
 something goes wrong. Here's a not-so-useful example:
@@ -116,8 +116,8 @@ the repository. Here's the full set of supported variables you can use here:
  * `output`: output of the command that failed (may be blank if an error
    occurred without running a command)
 
-Note that borgmatic runs the `on_error` hooks only for `prune`, `compact`,
-`create`, or `check` actions or hooks in which an error occurs, and not other
+Note that borgmatic runs the `on_error` hooks only for `create`, `prune`,
+`compact`, or `check` actions or hooks in which an error occurs, and not other
 actions. borgmatic does not run `on_error` hooks if an error occurs within a
 `before_everything` or `after_everything` hook. For more about hooks, see the
 [borgmatic hooks
@@ -144,7 +144,7 @@ With this hook in place, borgmatic pings your Healthchecks project when a
 backup begins, ends, or errors. Specifically, after the <a
 href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
 hooks</a> run, borgmatic lets Healthchecks know that it has started if any of
-the `prune`, `compact`, `create`, or `check` actions are run.
+the `create`, `prune`, `compact`, or `check` actions are run.
 
 Then, if the actions complete successfully, borgmatic notifies Healthchecks of
 the success after the `after_backup` hooks run, and includes borgmatic logs in
@@ -154,8 +154,8 @@ in the Healthchecks UI, although be aware that Healthchecks currently has a
 
 If an error occurs during any action or hook, borgmatic notifies Healthchecks
 after the `on_error` hooks run, also tacking on logs including the error
-itself. But the logs are only included for errors that occur when a `prune`,
-`compact`, `create`, or `check` action is run.
+itself. But the logs are only included for errors that occur when a `create`,
+`prune`, `compact`, or `check` action is run.
 
 You can customize the verbosity of the logs that are sent to Healthchecks with
 borgmatic's `--monitoring-verbosity` flag. The `--list` and `--stats` flags

docs/how-to/run-arbitrary-borg-commands.md

@@ -53,7 +53,8 @@ This runs Borg's `rlist` command once on each configured borgmatic repository.
 (The native `borgmatic rlist` action should be preferred for most use.)
 
 What if you only want to run Borg on a single configured borgmatic repository
-when you've got several configured? Not a problem.
+when you've got several configured? Not a problem. The `--repository` argument
+lets you specify the repository to use, either by its path or its label:
 
 ```bash
 borgmatic borg --repository repo.borg break-lock

docs/how-to/set-up-backups.md

@@ -90,10 +90,11 @@ installing borgmatic:
  * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/)
  * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/)
  * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic)
- * [OpenBSD](http://ports.su/sysutils/borgmatic)
+ * [OpenBSD](https://openports.pl/path/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
  * [macOS (via Homebrew)](https://formulae.brew.sh/formula/borgmatic)
  * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/)
+ * [NixOS](https://search.nixos.org/packages?show=borgmatic&sort=relevance&type=packages&query=borgmatic)
  * [Ansible role](https://github.com/borgbase/ansible-role-borgbackup)
  * [virtualenv](https://virtualenv.pypa.io/en/stable/)
 
@@ -156,7 +157,7 @@ variable or set the `BORG_PASSPHRASE` environment variable. See the
 section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption)
 of the Borg Quick Start for more info.
 
-Alternatively, you can specify the passphrase programatically by setting
+Alternatively, you can specify the passphrase programmatically by setting
 either the borgmatic `encryption_passcommand` configuration variable or the
 `BORG_PASSCOMMAND` environment variable. See the [Borg Security
 FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically)
@@ -179,6 +180,9 @@ following command is available for that:
 sudo validate-borgmatic-config
 ```
 
+You'll need to specify your configuration file with `--config` if it's not in
+a default location.
+
 This command's exit status (`$?` in Bash) is zero when configuration is valid
 and non-zero otherwise.
 

docs/how-to/upgrade.md

@@ -145,15 +145,18 @@ like this:
 ```yaml
 location:
     repositories:
-        - original.borg
+        - path: original.borg
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.10</span> Omit
+the `path:` portion of the `repositories` list.
+
 Change it to a new (not yet created) repository path:
 
 ```yaml
 location:
     repositories:
-        - upgraded.borg
+        - path: upgraded.borg
 ```
 
 Then, run the `rcreate` action (formerly `init`) to create that new Borg 2

docs/reference/command-line.md

@@ -7,8 +7,10 @@ eleventyNavigation:
 ---
 ## borgmatic options
 
-Here are all of the available borgmatic command-line options. This includes the separate options for
-each action sub-command:
+Here are all of the available borgmatic command-line options, including the
+separate options for each action sub-command. Note that most of the
+flags listed here do not have equivalents in borgmatic's [configuration
+file](https://torsion.org/borgmatic/docs/reference/configuration/).
 
 ```
 {% include borgmatic/command-line.txt %}

scripts/run-end-to-end-dev-tests

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# This script is for running end-to-end tests on a developer machine. It sets up database containers
+# to run tests against, runs the tests, and then tears down the containers.
+#
+# Run this script from the root directory of the borgmatic source.
+#
+# For more information, see:
+# https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
+
+set -e
+
+USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock
+
+if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then
+    export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH"
+fi
+
+docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \
+    --renew-anon-volumes --abort-on-container-exit

scripts/run-full-dev-tests

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# This script is for running all tests, including end-to-end tests, on a developer machine. It sets
-# up database containers to run tests against, runs the tests, and then tears down the containers.
-#
-# Run this script from the root directory of the borgmatic source.
-#
-# For more information, see:
-# https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
-
-set -e
-
-docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \
-    --renew-anon-volumes --abort-on-container-exit

scripts/run-full-tests

@@ -3,19 +3,30 @@
 # This script installs test dependencies and runs all tests, including end-to-end tests. It
 # is designed to run inside a test container, and presumes that other test infrastructure like
 # databases are already running. Therefore, on a developer machine, you should not run this script
-# directly. Instead, run scripts/run-full-dev-tests
+# directly. Instead, run scripts/run-end-to-end-dev-tests
 #
 # For more information, see:
 # https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
 
 set -e
 
+if [ -z "$TEST_CONTAINER" ] ; then
+    echo "This script is designed to work inside a test container and is not intended to"
+    echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute"
+    echo "scripts/run-end-to-end-dev-tests instead."
+    exit 1
+fi
+
 apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \
-    py3-ruamel.yaml py3-ruamel.yaml.clib bash
+    py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite
 # If certain dependencies of black are available in this version of Alpine, install them.
 apk add --no-cache py3-typed-ast py3-regex || true
 python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1
 pip3 install --ignore-installed tox==3.25.1
 export COVERAGE_FILE=/tmp/.coverage
-tox --workdir /tmp/.tox --sitepackages
+
+if [ "$1" != "--end-to-end-only" ] ; then
+    tox --workdir /tmp/.tox --sitepackages
+fi
+
 tox --workdir /tmp/.tox --sitepackages -e end-to-end

setup.cfg

@@ -5,11 +5,13 @@ description_file=README.md
 testpaths = tests
 addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end
 filterwarnings = 
-    ignore:Coverage disabled.*:pytest.PytestWarning
+    ignore:Deprecated call to `pkg_resources.declare_namespace\('ruamel'\)`.*:DeprecationWarning
 
 [flake8]
 ignore = E501,W503
 exclude = *.*/*
+multiline-quotes = '''
+docstring-quotes = '''
 
 [tool:isort]
 force_single_line = False
@@ -18,3 +20,6 @@ known_first_party = borgmatic
 line_length = 100
 multi_line_output = 3
 skip = .tox
+
+[codespell]
+skip = .git,.tox,build

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.7.8.dev0'
+VERSION = '1.7.12.dev0'
 
 
 setup(

test_requirements.txt

@@ -2,9 +2,13 @@ appdirs==1.4.4; python_version >= '3.8'
 attrs==20.3.0; python_version >= '3.8'
 black==19.10b0; python_version >= '3.8'
 click==7.1.2; python_version >= '3.8'
+codespell==2.2.4
 colorama==0.4.4
 coverage==5.3
 flake8==4.0.1
+flake8-quotes==3.3.2
+flake8-use-fstring==1.4
+flake8-variables-names==0.0.5
 flexmock==0.10.4
 isort==5.9.1
 mccabe==0.6.1

tests/end-to-end/docker-compose.yaml

@@ -1,30 +1,33 @@
 version: '3'
 services:
   postgresql:
-    image: postgres:13.1-alpine
+    image: docker.io/postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   mysql:
-    image: mariadb:10.5
+    image: docker.io/mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   mongodb:
-    image: mongo:5.0.5
+    image: docker.io/mongo:5.0.5
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
   tests:
-    image: alpine:3.13
+    image: docker.io/alpine:3.13
+    environment:
+      TEST_CONTAINER: true
     volumes:
       - "../..:/app:ro"
     tmpfs:
       - "/app/borgmatic.egg-info"
     tty: true
     working_dir: /app
-    command:
-      - /app/scripts/run-full-tests
+    entrypoint: /app/scripts/run-full-tests
+    command: --end-to-end-only
     depends_on:
       - postgresql
       - mysql
+      - mongodb

tests/end-to-end/test_borgmatic.py

@@ -12,17 +12,14 @@ def generate_configuration(config_path, repository_path):
     to work for testing (including injecting the given repository path and tacking on an encryption
     passphrase).
     '''
-    subprocess.check_call(
-        'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
-    )
+    subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
     config = (
         open(config_path)
         .read()
         .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
-        .replace('- ssh://user@backupserver/./{fqdn}', '')
-        .replace('- /var/local/backups/local.borg', '')
-        .replace('- /home/user/path with spaces', '')
-        .replace('- /home', '- {}'.format(config_path))
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home', f'- {config_path}')
         .replace('- /etc', '')
         .replace('- /var/log/syslog*', '')
         + 'storage:\n    encryption_passphrase: "test"'
@@ -47,13 +44,13 @@ def test_borgmatic_command():
         generate_configuration(config_path, repository_path)
 
         subprocess.check_call(
-            'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+            f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ')
         )
 
         # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
-        subprocess.check_call('borgmatic --config {}'.format(config_path).split(' '))
+        subprocess.check_call(f'borgmatic --config {config_path}'.split(' '))
         output = subprocess.check_output(
-            'borgmatic --config {} list --json'.format(config_path).split(' ')
+            f'borgmatic --config {config_path} list --json'.split(' ')
         ).decode(sys.stdout.encoding)
         parsed_output = json.loads(output)
 
@@ -64,16 +61,14 @@ def test_borgmatic_command():
         # Extract the created archive into the current (temporary) directory, and confirm that the
         # extracted file looks right.
         output = subprocess.check_output(
-            'borgmatic --config {} extract --archive {}'.format(config_path, archive_name).split(
-                ' '
-            )
+            f'borgmatic --config {config_path} extract --archive {archive_name}'.split(' '),
         ).decode(sys.stdout.encoding)
         extracted_config_path = os.path.join(extract_path, config_path)
         assert open(extracted_config_path).read() == open(config_path).read()
 
         # Exercise the info action.
         output = subprocess.check_output(
-            'borgmatic --config {} info --json'.format(config_path).split(' ')
+            f'borgmatic --config {config_path} info --json'.split(' '),
         ).decode(sys.stdout.encoding)
         parsed_output = json.loads(output)
 

tests/end-to-end/test_database.py

@@ -73,6 +73,9 @@ hooks:
           hostname: mongodb
           username: root
           password: test
+    sqlite_databases:
+        - name: sqlite_test
+          path: /tmp/sqlite_test.db
 '''
 
     with open(config_path, 'w') as config_file:
@@ -186,7 +189,7 @@ def test_database_dump_with_error_causes_borgmatic_to_exit():
                     '-v',
                     '2',
                     '--override',
-                    "hooks.postgresql_databases=[{'name': 'nope'}]",
+                    "hooks.postgresql_databases=[{'name': 'nope'}]",  # noqa: FS003
                 ]
             )
     finally:

tests/end-to-end/test_override.py

@@ -10,17 +10,15 @@ def generate_configuration(config_path, repository_path):
     to work for testing (including injecting the given repository path and tacking on an encryption
     passphrase).
     '''
-    subprocess.check_call(
-        'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
-    )
+    subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
     config = (
         open(config_path)
         .read()
         .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
-        .replace('- ssh://user@backupserver/./{fqdn}', '')
+        .replace('- ssh://user@backupserver/./{fqdn}', '')  # noqa: FS003
         .replace('- /var/local/backups/local.borg', '')
         .replace('- /home/user/path with spaces', '')
-        .replace('- /home', '- {}'.format(config_path))
+        .replace('- /home', f'- {config_path}')
         .replace('- /etc', '')
         .replace('- /var/log/syslog*', '')
         + 'storage:\n    encryption_passphrase: "test"'