Fix traceback that can occur when dumping a database (#440).

tests/integration/test_execute: use plain Python rather than xxd

.drone.yml

@@ -1,50 +1,78 @@
 ---
 kind: pipeline
-name: python-3-5-alpine-3-9
-
-steps:
-- name: build
-  image: python:3.5-alpine3.9
-  pull: always
-  commands:
-    - scripts/run-tests
----
-kind: pipeline
 name: python-3-6-alpine-3-9
 
-steps:
-- name: build
-  image: python:3.6-alpine3.9
-  pull: always
-  commands:
-    - scripts/run-tests
----
-kind: pipeline
-name: python-3-7-alpine-3-9
+services:
+  - name: postgresql
+    image: postgres:11.9-alpine
+    environment:
+      POSTGRES_PASSWORD: test
+      POSTGRES_DB: test
+  - name: mysql
+    image: mariadb:10.3
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_DATABASE: test
 
 steps:
 - name: build
-  image: python:3.7-alpine3.9
+  image: alpine:3.9
   pull: always
   commands:
-    - scripts/run-tests
+    - scripts/run-full-tests
 ---
 kind: pipeline
-name: python-3-7-alpine-3-7
+name: python-3-7-alpine-3-10
+
+services:
+  - name: postgresql
+    image: postgres:11.9-alpine
+    environment:
+      POSTGRES_PASSWORD: test
+      POSTGRES_DB: test
+  - name: mysql
+    image: mariadb:10.3
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_DATABASE: test
 
 steps:
 - name: build
-  image: python:3.7-alpine3.7
+  image: alpine:3.10
   pull: always
   commands:
-    - scripts/run-tests
+    - scripts/run-full-tests
+---
+kind: pipeline
+name: python-3-8-alpine-3-13
+
+services:
+  - name: postgresql
+    image: postgres:13.1-alpine
+    environment:
+      POSTGRES_PASSWORD: test
+      POSTGRES_DB: test
+  - name: mysql
+    image: mariadb:10.5
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_DATABASE: test
+
+steps:
+- name: build
+  image: alpine:3.13
+  pull: always
+  commands:
+    - scripts/run-full-tests
 ---
 kind: pipeline
 name: documentation
 
 steps:
 - name: build
-  image: plugins/docker
+  #image: plugins/docker
+  # Temporary work-around for https://github.com/drone-plugins/drone-docker/pull/327
+  image: techknowlogick/drone-docker
   settings:
     username:
       from_secret: docker_username
@@ -52,6 +80,7 @@ steps:
       from_secret: docker_password
     repo: witten/borgmatic-docs
     dockerfile: docs/Dockerfile
-  when:
-    branch:
-      - master
+
+trigger:
+  branch:
+    - master

.eleventy.js

@@ -1,9 +1,11 @@
 const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
 const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language");
+const navigationPlugin = require("@11ty/eleventy-navigation");
 
 module.exports = function(eleventyConfig) {
     eleventyConfig.addPlugin(pluginSyntaxHighlight);
     eleventyConfig.addPlugin(inclusiveLangPlugin);
+    eleventyConfig.addPlugin(navigationPlugin);
 
     let markdownIt = require("markdown-it");
     let markdownItAnchor = require("markdown-it-anchor");
@@ -13,10 +15,11 @@ module.exports = function(eleventyConfig) {
         html: true,
         breaks: false,
         linkify: true,
-        // Replace links to .md files with links to directories. This allows unparsed Markdown links
-        // to work on GitHub, while rendered links elsewhere also work.
         replaceLink: function (link, env) {
-            return link.replace(/\.md$/, '/');
+            if (process.env.NODE_ENV == "production") {
+                return link;
+            }
+            return link.replace('https://torsion.org/borgmatic/', 'http://localhost:8080/');
         }
     };
     let markdownItAnchorOptions = {
@@ -31,6 +34,8 @@ module.exports = function(eleventyConfig) {
             .use(markdownItReplaceLink)
     );
 
+    eleventyConfig.addPassthroughCopy({"docs/static": "static"});
+
     return {
         templateFormats: [
           "md",

.gitea/issue_template.md

@@ -26,6 +26,10 @@ Use `sudo borg --version`
 
 **Python version:** [version here]
 
-Use `python --version`
+Use `python3 --version`
+
+**Database version (if applicable):** [version here]
+
+Use `psql --version` or `mysql --version` on client and server.
 
 **operating system and version:** [OS here]

.gitignore

@@ -2,9 +2,10 @@
 *.pyc
 *.swp
 .cache
-.coverage
+.coverage*
 .pytest_cache
 .tox
+__pycache__
 build/
 dist/
 pip-wheel-metadata/

AUTHORS

@@ -10,3 +10,5 @@ newtonne: Read encryption password from external file
 Robin `ypid` Schneider: Support additional options of Borg and add validate-borgmatic-config command
 Scott Squires: Custom archive names
 Thomas LÉVEIL: Support for a keep_minutely prune option. Support for the --json option
+
+And many others! See the output of "git log".

MANIFEST.in

@@ -1 +1,2 @@
 include borgmatic/config/schema.yaml
+graft sample/systemd

NEWS

@@ -1,3 +1,352 @@
+1.5.18
+ * #389: Fix "message too long" error when logging to rsyslog.
+ * #440: Fix traceback that can occur when dumping a database.
+
+1.5.17
+ * #437: Fix error when configuration file contains "umask" option.
+ * Remove test dependency on vim and /dev/urandom.
+
+1.5.16
+ * #379: Suppress console output in sample crontab and systemd service files.
+ * #407: Fix syslog logging on FreeBSD.
+ * #430: Fix hang when restoring a PostgreSQL "tar" format database dump.
+ * Better error messages! Switch the library used for validating configuration files (from pykwalify
+   to jsonschema).
+ * Link borgmatic Ansible role from installation documentation:
+   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#other-ways-to-install
+
+1.5.15
+ * #419: Document use case of running backups conditionally based on laptop power level:
+   https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/
+ * #425: Run arbitrary Borg commands with new "borgmatic borg" action. See the documentation for
+   more information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/
+
+1.5.14
+ * #390: Add link to Hetzner storage offering from the documentation.
+ * #398: Clarify canonical home of borgmatic in documentation.
+ * #406: Clarify that spaces in path names should not be backslashed in path names.
+ * #423: Fix error handling to error loudly when Borg gets killed due to running out of memory!
+ * Fix build so as not to attempt to build and push documentation for a non-master branch.
+ * "Fix" build failure with Alpine Edge by switching from Edge to Alpine 3.13.
+ * Move #borgmatic IRC channel from Freenode to Libera Chat due to Freenode takeover drama.
+   IRC connection info: https://torsion.org/borgmatic/#issues
+
+1.5.13
+ * #373: Document that passphrase is used for Borg keyfile encryption, not just repokey encryption.
+ * #404: Add support for ruamel.yaml 0.17.x YAML parsing library.
+ * Update systemd service example to return a permission error when a system call isn't permitted
+   (instead of terminating borgmatic outright).
+ * Drop support for Python 3.5, which has been end-of-lifed.
+ * Add support for Python 3.9.
+ * Update versions of test dependencies (test_requirements.txt and test containers).
+ * Only support black code formatter on Python 3.8+. New black dependencies make installation
+   difficult on older versions of Python.
+ * Replace "improve this documentation" form with link to support and ticket tracker.
+
+1.5.12
+ * Fix for previous release with incorrect version suffix in setup.py. No other changes.
+
+1.5.11
+ * #341: Add "temporary_directory" option for changing Borg's temporary directory.
+ * #352: Lock down systemd security settings in sample systemd service file.
+ * #355: Fix traceback when a database hook value is null in a configuration file.
+ * #361: Merge override values when specifying the "--override" flag multiple times. The previous
+   behavior was to take the value of the last "--override" flag only.
+ * #367: Fix traceback when upgrading old INI-style configuration with upgrade-borgmatic-config.
+ * #368: Fix signal forwarding from borgmatic to Borg resulting in recursion traceback.
+ * #369: Document support for Borg placeholders in repository names.
+
+1.5.10
+ * #347: Add hooks that run for the "extract" action: "before_extract" and "after_extract".
+ * #350: Fix traceback when a configuration directory is non-readable due to directory permissions.
+ * Add documentation navigation links on left side of all documentation pages.
+ * Clarify documentation on configuration overrides, specifically the portion about list syntax:
+   http://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
+ * Clarify documentation overview of monitoring options:
+   http://torsion.org/borgmatic/docs/how-to/monitor-your-backups/
+
+1.5.9
+ * #300: Add "borgmatic export-tar" action to export an archive to a tar-formatted file or stream.
+ * #339: Fix for intermittent timing-related test failure of logging function.
+ * Clarify database documentation about excluding named pipes and character/block devices to prevent
+   hangs.
+ * Add documentation on how to make backups redundant with multiple repositories:
+   https://torsion.org/borgmatic/docs/how-to/make-backups-redundant/
+
+1.5.8
+ * #336: Fix for traceback when running Cronitor, Cronhub, and PagerDuty monitor hooks.
+
+1.5.7
+ * #327: Fix broken pass-through of BORG_* environment variables to Borg.
+ * #328: Fix duplicate logging to Healthchecks and send "after_*" hooks output to Healthchecks.
+ * #331: Add SSL support to PostgreSQL database configuration.
+ * #333: Fix for potential data loss (data not getting backed up) when borgmatic omitted configured
+   source directories in certain situations. Specifically, this occurred when two source directories
+   on different filesystems were related by parentage (e.g. "/foo" and "/foo/bar/baz") and the
+   one_file_system option was enabled.
+ * Update documentation code fragments theme to better match the rest of the page.
+ * Improve configuration reference documentation readability via more aggressive word-wrapping in
+   configuration schema descriptions.
+
+1.5.6
+ * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the
+   monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring
+   services with a "start" status until after before_* hooks finish. Failures in before_* hooks
+   still trigger a monitoring "fail" status.
+ * #316: Fix hang when a stale database dump named pipe from an aborted borgmatic run remains on
+   disk.
+ * #323: Fix for certain configuration options like ssh_command impacting Borg invocations for
+   separate configuration files.
+ * #324: Add "borgmatic extract --strip-components" flag to remove leading path components when
+   extracting an archive.
+ * Tweak comment indentation in generated configuration file for clarity.
+ * Link to Borgmacator GNOME AppIndicator from monitoring documentation.
+
+1.5.5
+ * #314: Fix regression in support for PostgreSQL's "directory" dump format. Unlike other dump
+   formats, the "directory" dump format does not stream directly to/from Borg.
+ * #315: Fix enabled database hooks to implicitly set one_file_system configuration option to true.
+   This prevents Borg from reading devices like /dev/zero and hanging.
+ * #316: Fix hang when streaming a database dump to Borg with implicit duplicate source directories
+   by deduplicating them first.
+ * #319: Fix error message when there are no MySQL databases to dump for "all" databases.
+ * Improve documentation around the installation process. Specifically, making borgmatic commands
+   runnable via the system PATH and offering a global install option.
+
+1.5.4
+ * #310: Fix legitimate database dump command errors (exit code 1) not being treated as errors by
+   borgmatic.
+ * For database dumps, replace the named pipe on every borgmatic run. This prevent hangs on stale
+   pipes left over from previous runs.
+ * Fix error handling to handle more edge cases when executing commands.
+
+1.5.3
+ * #258: Stream database dumps and restores directly to/from Borg without using any additional
+   filesystem space. This feature is automatic, and works even on restores from archives made with
+   previous versions of borgmatic.
+ * #293: Documentation on macOS launchd permissions issues with work-around for Full Disk Access.
+ * Remove "borgmatic restore --progress" flag, as it now conflicts with streaming database restores.
+
+1.5.2
+ * #301: Fix MySQL restore error on "all" database dump by excluding system tables.
+ * Fix PostgreSQL restore error on "all" database dump by using "psql" for the restore instead of
+   "pg_restore".
+
+1.5.1
+ * #289: Tired of looking up the latest successful archive name in order to pass it to borgmatic
+   actions? Me too. Now you can specify "--archive latest" to all actions that accept an archive
+   flag.
+ * #290: Fix the "--stats" and "--files" flags so that they yield output at verbosity 0.
+ * Reduce the default verbosity of borgmatic logs sent to Healthchecks monitoring hook. Now, it's
+   warnings and errors only. You can increase the verbosity via the "--monitoring-verbosity" flag.
+ * Add security policy documentation in SECURITY.md.
+
+1.5.0
+ * #245: Monitor backups with PagerDuty hook integration. See the documentation for more
+   information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook
+ * #255: Add per-action hooks: "before_prune", "after_prune", "before_check", and "after_check".
+ * #274: Add ~/.config/borgmatic.d as another configuration directory default.
+ * #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag.
+ * #280: Change "exclude_if_present" option to support multiple filenames that indicate a directory
+   should be excluded from backups, rather than just a single filename.
+ * #284: Backup to a removable drive or intermittent server via "soft failure" feature. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/
+ * #287: View consistency check progress via "--progress" flag for "check" action.
+ * For "create" and "prune" actions, no longer list files or show detailed stats at any verbosities
+   by default. You can opt back in with "--files" or "--stats" flags.
+ * For "list" and "info" actions, show repository names even at verbosity 0.
+
+1.4.22
+ * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON ouput.
+ * After a backup of a database dump in directory format, properly remove the dump directory.
+ * In "borgmatic --help", don't expand $HOME in listing of default "--config" paths.
+
+1.4.21
+ * #268: Override particular configuration options from the command-line via "--override" flag. See
+   the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
+ * #270: Only trigger "on_error" hooks and monitoring failures for "prune", "create", and "check"
+   actions, and not for other actions.
+ * When pruning with verbosity level 1, list pruned and kept archives. Previously, this information
+   was only shown at verbosity level 2.
+
+1.4.20
+ * Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option.
+ * #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and
+   "prune" actions, not just "create".
+
+1.4.19
+ * #259: Optionally change the internal database dump path via "borgmatic_source_directory" option
+   in location configuration section.
+ * #271: Support piping "borgmatic list" output to grep by logging certain log levels to console
+   stdout and others to stderr.
+ * Retain colored output when piping or redirecting in an interactive terminal.
+ * Add end-to-end tests for database dump and restore. These are run on developer machines with
+   Docker Compose for approximate parity with continuous integration tests.
+
+1.4.18
+ * Fix "--repository" flag to accept relative paths.
+ * Fix "borgmatic umount" so it only runs Borg once instead of once per repository / configuration
+   file.
+ * #253: Mount whole repositories via "borgmatic mount" without any "--archive" flag.
+ * #269: Filter listed paths via "borgmatic list --path" flag.
+
+1.4.17
+ * #235: Pass extra options directly to particular Borg commands, handy for Borg options that
+   borgmatic does not yet support natively. Use "extra_borg_options" in the storage configuration
+   section.
+ * #266: Attempt to repair any inconsistencies found during a consistency check via
+   "borgmatic check --repair" flag.
+
+1.4.16
+ * #256: Fix for "before_backup" hook not triggering an error when the command contains "borg" and
+   has an exit code of 1.
+ * #257: Fix for garbled Borg file listing when using "borgmatic create --progress" with
+   verbosity level 1 or 2.
+ * #260: Fix for missing Healthchecks monitoring payload or HTTP 500 due to incorrect unicode
+   encoding.
+
+1.4.15
+ * Fix for database dump removal incorrectly skipping some database dumps.
+ * #123: Support for mounting an archive as a FUSE filesystem via "borgmatic mount" action, and
+   unmounting via "borgmatic umount". See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#mount-a-filesystem
+
+1.4.14
+ * Show summary log errors regardless of verbosity level, and log the "summary:" header with a log
+   level based on the contained summary logs.
+
+1.4.13
+ * Show full error logs at "--verbosity 0" so you can see command output without upping the
+   verbosity level.
+
+1.4.12
+ * #247: With "borgmatic check", consider Borg warnings as errors.
+ * Dial back the display of inline error logs a bit, so failed command output doesn't appear
+   multiple times in the logs (well, except for the summary).
+
+1.4.11
+ * #241: When using the Healthchecks monitoring hook, include borgmatic logs in the payloads for
+   completion and failure pings.
+ * With --verbosity level 1 or 2, show error logs both inline when they occur and in the summary
+   logs at the bottom. With lower verbosity levels, suppress the summary and show error logs when
+   they occur.
+
+1.4.10
+ * #246: Fix for "borgmatic restore" showing success and incorrectly extracting archive files, even
+   when no databases are configured to restore. As this can overwrite files from the archive and
+   lead to data loss, please upgrade to get the fix before using "borgmatic restore".
+ * Reopen the file given by "--log-file" flag if an external program rotates the log file while
+   borgmatic is running.
+
+1.4.9
+ * #228: Database dump hooks for MySQL/MariaDB, so you can easily dump your databases before backups
+   run.
+ * #243: Fix repository does not exist error with "borgmatic extract" when repository is remote.
+
+1.4.8
+ * Monitor backups with Cronhub hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook
+ * Fix Healthchecks/Cronitor hooks to skip actions when the borgmatic "--dry-run" flag is used.
+
+1.4.7
+ * #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks.
+ * #239: Upgrade your borgmatic configuration to get new options and comments via
+   "generate-borgmatic-config --source". See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your-configuration
+
+1.4.6
+ * Verbosity level "-1" for even quieter output: Errors only (#236).
+
+1.4.5
+ * Log to file instead of syslog via command-line "--log-file" flag (#233).
+
+1.4.4
+ * #234: Support for Borg --keep-exclude-tags and --exclude-nodump options.
+
+1.4.3
+ * Monitor backups with Cronitor hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
+
+1.4.2
+ * Extract files to a particular directory via "borgmatic extract --destination" flag.
+ * Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate
+   "borgmatic restore" action. Any uses of "--restore-path" will continue working.
+
+1.4.1
+ * #229: Restore backed up PostgreSQL databases via "borgmatic restore" action. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/
+ * Documentation on how to develop borgmatic's documentation:
+   https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/#documentation-development
+
+1.4.0
+ * #225: Database dump hooks for PostgreSQL, so you can easily dump your databases before backups
+   run.
+ * #230: Rename "borgmatic list --pattern-from" flag to "--patterns-from" to match Borg.
+
+1.3.26
+ * #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
+   (non-checkpoint) archives.
+
+1.3.25
+ * #223: Dead man's switch to detect when backups start failing silently, implemented via
+   healthchecks.io hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
+ * Documentation on monitoring and alerting options for borgmatic backups:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/
+ * Automatically rewrite links when developing on documentation locally.
+
+1.3.24
+ * #86: Add "borgmatic list --successful" flag to only list successful (non-checkpoint) archives.
+ * Add a suggestion form to all documentation pages, so users can submit ideas for improving the
+   documentation.
+ * Update documentation link to community Arch Linux borgmatic package.
+
+1.3.23
+ * #174: More detailed error alerting via runtime context available in "on_error" hook.
+
+1.3.22
+ * #144: When backups to one of several repositories fails, keep backing up to the other
+   repositories and report errors afterwards.
+
+1.3.21
+ * #192: User-defined hooks for global setup or cleanup that run before/after all actions. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+
+1.3.20
+ * #205: More robust sample systemd service: boot delay, network dependency, lowered CPU/IO
+   priority, etc.
+ * #221: Fix "borgmatic create --progress" output so that it updates on the console in real-time.
+
+1.3.19
+ * #219: Fix visibility of "borgmatic prune --stats" output.
+
+1.3.18
+ * #220: Fix regression of argument parsing for default actions.
+
+1.3.17
+ * #217: Fix error with "borgmatic check --only" command-line flag with "extract" consistency check.
+
+1.3.16
+ * #210: Support for Borg check --verify-data flag via borgmatic "data" consistency check.
+ * #210: Override configured consistency checks via "borgmatic check --only" command-line flag.
+ * When generating sample configuration with generate-borgmatic-config, add a space after each "#"
+   comment indicator.
+
+1.3.15
+ * #208: Fix for traceback when the "checks" option has an empty value.
+ * #209: Bypass Borg error about a moved repository via "relocated_repo_access_is_ok" option in
+   borgmatic storage configuration section.
+ * #213: Reorder arguments passed to Borg to fix duplicate directories when using Borg patterns.
+ * #214: Fix for hook erroring with exit code 1 not being interpreted as an error.
+
+1.3.14
+ * #204: Do not treat Borg warnings (exit code 1) as failures.
+ * When validating configuration files, require strings instead of allowing any scalar type.
+
 1.3.13
  * #199: Add note to documentation about using spaces instead of tabs for indentation, as YAML does
    not allow tabs.

README.md

@@ -2,91 +2,105 @@
 title: borgmatic
 permalink: index.html
 ---
-<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master)</a>
 
-## Overview
+## It's your data. Keep it that way.
 
-<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
+<img src="docs/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
 
 borgmatic is simple, configuration-driven backup software for servers and
-workstations. Backup all of your machines from the command-line or scheduled
-jobs. No GUI required. Built atop [Borg Backup](https://www.borgbackup.org/),
-borgmatic initiates a backup, prunes any old backups according to a retention
-policy, and validates backups for consistency. borgmatic supports specifying
-your settings in a declarative configuration file, rather than having to put
-them all on the command-line, and handles common errors.
+workstations. Protect your files with client-side encryption. Backup your
+databases too. Monitor it all with integrated third-party services.
 
-Here's an example config file:
+The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic</a>.
+
+Here's an example configuration file:
 
 ```yaml
 location:
-    # List of source directories to backup. Globs are expanded.
+    # List of source directories to backup.
     source_directories:
         - /home
         - /etc
-        - /var/log/syslog*
 
-    # Paths to local or remote repositories.
+    # Paths of local or remote repositories to backup to.
     repositories:
-        - user@backupserver:sourcehostname.borg
-
-    # Any paths matching these patterns are excluded from backups.
-    exclude_patterns:
-        - /home/*/.cache
+        - 1234@usw-s001.rsync.net:backups.borg
+        - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
+        - user1@scp2.cdn.lima-labs.com:repo
+        - /var/lib/backups/local.borg
 
 retention:
-    # Retention policy for how many backups to keep in each category.
+    # Retention policy for how many backups to keep.
     keep_daily: 7
     keep_weekly: 4
     keep_monthly: 6
 
 consistency:
-    # List of consistency checks to run: "repository", "archives", or both.
+    # List of checks to run to validate your backups.
     checks:
         - repository
         - archives
-```
 
-borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
-available](https://projects.torsion.org/witten/borgmatic). It's also mirrored
-on [GitHub](https://github.com/witten/borgmatic) for convenience.
+hooks:
+    # Custom preparation scripts to run.
+    before_backup:
+        - prepare-for-backup.sh
+
+    # Databases to dump and include in backups.
+    postgresql_databases:
+        - name: users
+
+    # Third-party services to notify you if backups aren't happening.
+    healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
+```
 
 Want to see borgmatic in action? Check out the <a
 href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
 
 <script src="https://asciinema.org/a/203761.js" id="asciicast-203761" async></script>
 
+borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 
-## How-to guides
+## Integrations
 
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/) ⬅ *Start here!*
- * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
- * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
- * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
- * [Restore a backup](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/)
- * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
- * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
- * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
+<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic"><img src="docs/static/rsyncnet.png" alt="rsync.net" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 
 
-## Reference guides
+## Getting started
 
- * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)
- * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)
+Your first step is to [install and configure
+borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/).
+
+For additional documentation, check out the links above for <a
+href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
+reference guides</a>.
 
 
 ## Hosting providers
 
-Need somewhere to store your encrypted offsite backups? The following hosting
-providers include specific support for Borg/borgmatic. Using these links and
-services helps support borgmatic development and hosting. (These are referral
-links, but without any tracking scripts or cookies.)
+Need somewhere to store your encrypted off-site backups? The following hosting
+providers include specific support for Borg/borgmatic—and fund borgmatic
+development and hosting when you use these links to sign up. (These are
+referral links, but without any tracking scripts or cookies.)
 
 <ul>
  <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
  <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+ <li class="referral"><a href="https://storage.lima-labs.com/special-pricing-offer-for-borgmatic-users/">Lima-Labs</a>: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs</li>
 </ul>
 
+Additionally, [Hetzner](https://www.hetzner.com/storage/storage-box) has a
+compatible storage offering, but does not currently fund borgmatic
+development or hosting.
+
 ## Support and contributing
 
 ### Issues
@@ -98,15 +112,29 @@ first](https://projects.torsion.org/user/login). Note that you can login with
 an existing GitHub account if you prefer.
 
 If you'd like to chat with borgmatic developers or users, head on over to the
-`#borgmatic` IRC channel on Freenode, either via <a
-href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
-native <a href="irc://chat.freenode.net:6697">IRC client</a>.
+`#borgmatic` IRC channel on Libera Chat, either via <a
+href="https://web.libera.chat/#borgmatic">web chat</a> or a
+native <a href="ircs://irc.libera.chat:6697">IRC client</a>. If you
+don't get a response right away, please hang around a while—or file a ticket
+instead.
 
-Other questions or comments? Contact <mailto:witten@torsion.org>.
+Also see the [security
+policy](https://torsion.org/borgmatic/docs/security-policy/) for any security
+issues.
+
+Other questions or comments? Contact
+[witten@torsion.org](mailto:witten@torsion.org).
 
 
 ### Contributing
 
+borgmatic [source code is
+available](https://projects.torsion.org/witten/borgmatic) and is also mirrored
+on [GitHub](https://github.com/witten/borgmatic) for convenience.
+
+borgmatic is licensed under the GNU General Public License version 3 or any
+later version.
+
 If you'd like to contribute to borgmatic development, please feel free to
 submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls)
 or open an [issue](https://projects.torsion.org/witten/borgmatic/issues) first
@@ -117,7 +145,5 @@ Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
 
-<script>
-  var links = document.getElementsByClassName("referral");
-  links[Math.floor(Math.random() * links.length)].style.display = "none";
-</script>
+<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master)</a>
+

SECURITY.md

@@ -0,0 +1,18 @@
+---
+title: Security policy
+permalink: security-policy/index.html
+---
+
+## Supported versions
+
+While we want to hear about security vulnerabilities in all versions of
+borgmatic, security fixes are only made to the most recently released version.
+It's simply not practical for our small volunteer effort to maintain multiple
+release branches and put out separate security patches for each.
+
+## Reporting a vulnerability
+
+If you find a security vulnerability, please [file a
+ticket](https://torsion.org/borgmatic/#issues) or [send email
+directly](mailto:witten@torsion.org) as appropriate. You should expect to hear
+back within a few days at most and generally sooner.

borgmatic/borg/borg.py

@@ -0,0 +1,45 @@
+import logging
+
+from borgmatic.borg.flags import make_flags
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+REPOSITORYLESS_BORG_COMMANDS = {'serve', None}
+
+
+def run_arbitrary_borg(
+    repository, storage_config, options, archive=None, local_path='borg', remote_path=None
+):
+    '''
+    Given a local or remote repository path, a storage config dict, a sequence of arbitrary
+    command-line Borg options, and an optional archive name, run an arbitrary Borg command on the
+    given repository/archive.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+
+    try:
+        options = options[1:] if options[0] == '--' else options
+        borg_command = options[0]
+        command_options = tuple(options[1:])
+    except IndexError:
+        borg_command = None
+        command_options = ()
+
+    repository_archive = '::'.join((repository, archive)) if repository and archive else repository
+
+    full_command = (
+        (local_path,)
+        + ((borg_command,) if borg_command else ())
+        + ((repository_archive,) if borg_command and repository_archive else ())
+        + command_options
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+    )
+
+    return execute_command(
+        full_command, output_log_level=logging.WARNING, borg_local_path=local_path,
+    )

borgmatic/borg/check.py

@@ -1,7 +1,7 @@
 import logging
 
 from borgmatic.borg import extract
-from borgmatic.execute import execute_command
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 DEFAULT_CHECKS = ('repository', 'archives')
 DEFAULT_PREFIX = '{hostname}-'
@@ -10,9 +10,10 @@ DEFAULT_PREFIX = '{hostname}-'
 logger = logging.getLogger(__name__)
 
 
-def _parse_checks(consistency_config):
+def _parse_checks(consistency_config, only_checks=None):
     '''
-    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
+    Given a consistency config with a "checks" list, and an optional list of override checks,
+    transform them a tuple of named checks to run.
 
     For example, given a retention config of:
 
@@ -22,16 +23,21 @@ def _parse_checks(consistency_config):
 
         ('repository', 'archives')
 
-    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
-    "disabled", return an empty tuple, meaning that no checks should be run.
+    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If the checks value
+    is the string "disabled", return an empty tuple, meaning that no checks should be run.
+
+    If the "data" option is present, then make sure the "archives" option is included as well.
     '''
-    checks = consistency_config.get('checks', [])
+    checks = [
+        check.lower() for check in (only_checks or consistency_config.get('checks', []) or [])
+    ]
     if checks == ['disabled']:
         return ()
 
-    return (
-        tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
-    )
+    if 'data' in checks and 'archives' not in checks:
+        checks.append('archives')
+
+    return tuple(check for check in checks if check not in ('disabled', '')) or DEFAULT_CHECKS
 
 
 def _make_check_flags(checks, check_last=None, prefix=None):
@@ -68,33 +74,42 @@ def _make_check_flags(checks, check_last=None, prefix=None):
                 'Ignoring consistency prefix option, as "archives" is not in consistency checks.'
             )
 
+    common_flags = last_flags + prefix_flags + (('--verify-data',) if 'data' in checks else ())
+
     if set(DEFAULT_CHECKS).issubset(set(checks)):
-        return last_flags + prefix_flags
+        return common_flags
 
     return (
         tuple('--{}-only'.format(check) for check in checks if check in DEFAULT_CHECKS)
-        + last_flags
-        + prefix_flags
+        + common_flags
     )
 
 
 def check_archives(
-    repository, storage_config, consistency_config, local_path='borg', remote_path=None
+    repository,
+    storage_config,
+    consistency_config,
+    local_path='borg',
+    remote_path=None,
+    progress=None,
+    repair=None,
+    only_checks=None,
 ):
     '''
     Given a local or remote repository path, a storage config dict, a consistency config dict,
-    and a local/remote commands to run, check the contained Borg archives for consistency.
+    local/remote commands to run, whether to include progress information, whether to attempt a
+    repair, and an optional list of checks to use instead of configured checks, check the contained
+    Borg archives for consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
-    checks = _parse_checks(consistency_config)
+    checks = _parse_checks(consistency_config, only_checks)
     check_last = consistency_config.get('check_last', None)
     lock_wait = None
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
 
-    if set(checks).intersection(set(DEFAULT_CHECKS)):
-        remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
         lock_wait = storage_config.get('lock_wait', None)
-        lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
 
         verbosity_flags = ()
         if logger.isEnabledFor(logging.INFO):
@@ -105,14 +120,23 @@ def check_archives(
         prefix = consistency_config.get('prefix', DEFAULT_PREFIX)
 
         full_command = (
-            (local_path, 'check', repository)
+            (local_path, 'check')
+            + (('--repair',) if repair else ())
             + _make_check_flags(checks, check_last, prefix)
-            + remote_path_flags
-            + lock_wait_flags
+            + (('--remote-path', remote_path) if remote_path else ())
+            + (('--lock-wait', str(lock_wait)) if lock_wait else ())
             + verbosity_flags
+            + (('--progress',) if progress else ())
+            + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+            + (repository,)
         )
 
-        execute_command(full_command)
+        # The Borg repair option trigger an interactive prompt, which won't work when output is
+        # captured. And progress messes with the terminal directly.
+        if repair or progress:
+            execute_command(full_command, output_file=DO_NOT_CAPTURE)
+        else:
+            execute_command(full_command)
 
     if 'extract' in checks:
         extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -2,9 +2,10 @@ import glob
 import itertools
 import logging
 import os
+import pathlib
 import tempfile
 
-from borgmatic.execute import execute_command
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +44,53 @@ def _expand_home_directories(directories):
     return tuple(os.path.expanduser(directory) for directory in directories)
 
 
+def map_directories_to_devices(directories):  # pragma: no cover
+    '''
+    Given a sequence of directories, return a map from directory to an identifier for the device on
+    which that directory resides. This is handy for determining whether two different directories
+    are on the same filesystem (have the same device identifier).
+    '''
+    return {directory: os.stat(directory).st_dev for directory in directories}
+
+
+def deduplicate_directories(directory_devices):
+    '''
+    Given a map from directory to the identifier for the device on which that directory resides,
+    return the directories as a sorted tuple with all duplicate child directories removed. For
+    instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
+
+    The one exception to this rule is if two paths are on different filesystems (devices). In that
+    case, they won't get de-duplicated in case they both need to be passed to Borg (e.g. the
+    location.one_file_system option is true).
+
+    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
+    child directories, because it will naturally spider the contents of the parent directory. And
+    there are cases where Borg coming across the same file twice will result in duplicate reads and
+    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
+    Borg.
+    '''
+    deduplicated = set()
+    directories = sorted(directory_devices.keys())
+
+    for directory in directories:
+        deduplicated.add(directory)
+        parents = pathlib.PurePath(directory).parents
+
+        # If another directory in the given list is a parent of current directory (even n levels
+        # up) and both are on the same filesystem, then the current directory is a duplicate.
+        for other_directory in directories:
+            for parent in parents:
+                if (
+                    pathlib.PurePath(other_directory) == parent
+                    and directory_devices[other_directory] == directory_devices[directory]
+                ):
+                    if directory in deduplicated:
+                        deduplicated.remove(directory)
+                    break
+
+    return tuple(sorted(deduplicated))
+
+
 def _write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -60,8 +108,8 @@ def _write_pattern_file(patterns=None):
 
 def _make_pattern_flags(location_config, pattern_filename=None):
     '''
-    Given a location config dict with a potential pattern_from option, and a filename containing any
-    additional patterns, return the corresponding Borg flags for those files as a tuple.
+    Given a location config dict with a potential patterns_from option, and a filename containing
+    any additional patterns, return the corresponding Borg flags for those files as a tuple.
     '''
     pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
         (pattern_filename,) if pattern_filename else ()
@@ -88,10 +136,44 @@ def _make_exclude_flags(location_config, exclude_filename=None):
         )
     )
     caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
-    if_present = location_config.get('exclude_if_present')
-    if_present_flags = ('--exclude-if-present', if_present) if if_present else ()
+    if_present_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-if-present', if_present)
+            for if_present in location_config.get('exclude_if_present', ())
+        )
+    )
+    keep_exclude_tags_flags = (
+        ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else ()
+    )
+    exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else ()
 
-    return exclude_from_flags + caches_flag + if_present_flags
+    return (
+        exclude_from_flags
+        + caches_flag
+        + if_present_flags
+        + keep_exclude_tags_flags
+        + exclude_nodump_flags
+    )
+
+
+DEFAULT_BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
+
+
+def borgmatic_source_directories(borgmatic_source_directory):
+    '''
+    Return a list of borgmatic-specific source directories used for state like database backups.
+    '''
+    if not borgmatic_source_directory:
+        borgmatic_source_directory = DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+
+    return (
+        [borgmatic_source_directory]
+        if os.path.exists(os.path.expanduser(borgmatic_source_directory))
+        else []
+    )
+
+
+DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
 
 
 def create_archive(
@@ -104,12 +186,24 @@ def create_archive(
     progress=False,
     stats=False,
     json=False,
+    files=False,
+    stream_processes=None,
 ):
     '''
     Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
     storage config dict, create a Borg archive and return Borg's JSON output (if any).
+
+    If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
+    create command while also triggering the given processes to produce output.
     '''
-    sources = _expand_directories(location_config['source_directories'])
+    sources = deduplicate_directories(
+        map_directories_to_devices(
+            _expand_directories(
+                location_config['source_directories']
+                + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+            )
+        )
+    )
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
     exclude_file = _write_pattern_file(
@@ -122,53 +216,67 @@ def create_archive(
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
     files_cache = location_config.get('files_cache')
-    default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
-    archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
+    archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
 
     full_command = (
-        (
-            local_path,
-            'create',
-            '{repository}::{archive_name_format}'.format(
-                repository=repository, archive_name_format=archive_name_format
-            ),
-        )
-        + sources
+        tuple(local_path.split(' '))
+        + ('create',)
         + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
         + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
         + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
-        + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (
+            ('--one-file-system',)
+            if location_config.get('one_file_system') or stream_processes
+            else ()
+        )
         + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--noatime',) if location_config.get('atime') is False else ())
         + (('--noctime',) if location_config.get('ctime') is False else ())
         + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
-        + (('--read-special',) if location_config.get('read_special') else ())
+        + (('--read-special',) if (location_config.get('read_special') or stream_processes) else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--list', '--filter', 'AME-') if logger.isEnabledFor(logging.INFO) and not json else ())
+        + (('--list', '--filter', 'AME-') if files and not json and not progress else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
-        + (
-            ('--stats',)
-            if not dry_run and (logger.isEnabledFor(logging.INFO) or stats) and not json
-            else ()
-        )
+        + (('--stats',) if stats and not json and not dry_run else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
         + (('--dry-run',) if dry_run else ())
         + (('--progress',) if progress else ())
         + (('--json',) if json else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (
+            '{repository}::{archive_name_format}'.format(
+                repository=repository, archive_name_format=archive_name_format
+            ),
+        )
+        + sources
     )
 
     if json:
         output_log_level = None
-    elif stats:
+    elif (stats or files) and logger.getEffectiveLevel() == logging.WARNING:
         output_log_level = logging.WARNING
     else:
         output_log_level = logging.INFO
 
-    return execute_command(full_command, output_log_level)
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    output_file = DO_NOT_CAPTURE if progress else None
+
+    if stream_processes:
+        return execute_command_with_processes(
+            full_command,
+            stream_processes,
+            output_log_level,
+            output_file,
+            borg_local_path=local_path,
+        )
+
+    return execute_command(full_command, output_log_level, output_file, borg_local_path=local_path)

borgmatic/borg/environment.py

@@ -9,11 +9,30 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'encryption_passcommand': 'BORG_PASSCOMMAND',
     'encryption_passphrase': 'BORG_PASSPHRASE',
     'ssh_command': 'BORG_RSH',
+    'temporary_directory': 'TMPDIR',
+}
+
+DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
+    'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
 }
 
 
 def initialize(storage_config):
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
-        value = storage_config.get(option_name)
+
+        # Options from borgmatic configuration take precedence over already set BORG_* environment
+        # variables.
+        value = storage_config.get(option_name) or os.environ.get(environment_variable_name)
+
         if value:
             os.environ[environment_variable_name] = value
+        else:
+            os.environ.pop(environment_variable_name, None)
+
+    for (
+        option_name,
+        environment_variable_name,
+    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name, False)
+        os.environ[environment_variable_name] = 'yes' if value else 'no'

borgmatic/borg/export_tar.py

@@ -0,0 +1,64 @@
+import logging
+import os
+
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def export_tar_archive(
+    dry_run,
+    repository,
+    archive,
+    paths,
+    destination_path,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+    tar_filter=None,
+    files=False,
+    strip_components=None,
+):
+    '''
+    Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
+    export from the archive, a destination path to export to, a storage configuration dict, optional
+    local and remote Borg paths, an optional filter program, whether to include per-file details,
+    and an optional number of path components to strip, export the archive into the given
+    destination path as a tar-formatted file.
+
+    If the destination path is "-", then stream the output to stdout instead of to a file.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'export-tar')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--list',) if files else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--tar-filter', tar_filter) if tar_filter else ())
+        + (('--strip-components', str(strip_components)) if strip_components else ())
+        + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
+        + (destination_path,)
+        + (tuple(paths) if paths else ())
+    )
+
+    if files and logger.getEffectiveLevel() == logging.WARNING:
+        output_log_level = logging.WARNING
+    else:
+        output_log_level = logging.INFO
+
+    if dry_run:
+        logging.info('{}: Skipping export to tar file (dry run)'.format(repository))
+        return
+
+    execute_command(
+        full_command,
+        output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
+        output_log_level=output_log_level,
+        borg_local_path=local_path,
+    )

borgmatic/borg/extract.py

@@ -1,6 +1,8 @@
 import logging
+import os
+import subprocess
 
-from borgmatic.execute import execute_command
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -19,13 +21,16 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         verbosity_flags = ('--info',)
 
     full_list_command = (
-        (local_path, 'list', '--short', repository)
+        (local_path, 'list', '--short')
         + remote_path_flags
         + lock_wait_flags
         + verbosity_flags
+        + (repository,)
     )
 
-    list_output = execute_command(full_list_command, output_log_level=None)
+    list_output = execute_command(
+        full_list_command, output_log_level=None, borg_local_path=local_path
+    )
 
     try:
         last_archive_name = list_output.strip().splitlines()[-1]
@@ -34,45 +39,52 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
     full_extract_command = (
-        (
-            local_path,
-            'extract',
-            '--dry-run',
-            '{repository}::{last_archive_name}'.format(
-                repository=repository, last_archive_name=last_archive_name
-            ),
-        )
+        (local_path, 'extract', '--dry-run')
         + remote_path_flags
         + lock_wait_flags
         + verbosity_flags
         + list_flag
+        + (
+            '{repository}::{last_archive_name}'.format(
+                repository=repository, last_archive_name=last_archive_name
+            ),
+        )
     )
 
-    execute_command(full_extract_command)
+    execute_command(full_extract_command, working_directory=None)
 
 
 def extract_archive(
     dry_run,
     repository,
     archive,
-    restore_paths,
+    paths,
     location_config,
     storage_config,
     local_path='borg',
     remote_path=None,
+    destination_path=None,
+    strip_components=None,
     progress=False,
+    extract_to_stdout=False,
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
-    restore from the archive, and location/storage configuration dicts, extract the archive into the
-    current directory.
+    restore from the archive, location/storage configuration dicts, optional local and remote Borg
+    paths, and an optional destination path to extract to, extract the archive into the current
+    directory.
+
+    If extract to stdout is True, then start the extraction streaming to stdout, and return that
+    extract process as an instance of subprocess.Popen.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
 
+    if progress and extract_to_stdout:
+        raise ValueError('progress and extract_to_stdout cannot both be set')
+
     full_command = (
-        (local_path, 'extract', '::'.join((repository, archive)))
-        + (tuple(restore_paths) if restore_paths else ())
+        (local_path, 'extract')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--umask', str(umask)) if umask else ())
@@ -80,7 +92,29 @@ def extract_archive(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
+        + (('--strip-components', str(strip_components)) if strip_components else ())
         + (('--progress',) if progress else ())
+        + (('--stdout',) if extract_to_stdout else ())
+        + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
+        + (tuple(paths) if paths else ())
     )
 
-    execute_command(full_command)
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if progress:
+        return execute_command(
+            full_command, output_file=DO_NOT_CAPTURE, working_directory=destination_path
+        )
+        return None
+
+    if extract_to_stdout:
+        return execute_command(
+            full_command,
+            output_file=subprocess.PIPE,
+            working_directory=destination_path,
+            run_to_completion=False,
+        )
+
+    # Don't give Borg local path, so as to error on warnings, as Borg only gives a warning if the
+    # restore paths don't exist in the archive!
+    execute_command(full_command, working_directory=destination_path)

borgmatic/borg/info.py

@@ -17,13 +17,7 @@ def display_archives_info(
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (
-            local_path,
-            'info',
-            '::'.join((repository, info_arguments.archive))
-            if info_arguments.archive
-            else repository,
-        )
+        (local_path, 'info')
         + (
             ('--info',)
             if logger.getEffectiveLevel() == logging.INFO and not info_arguments.json
@@ -37,8 +31,15 @@ def display_archives_info(
         + make_flags('remote-path', remote_path)
         + make_flags('lock-wait', lock_wait)
         + make_flags_from_arguments(info_arguments, excludes=('repository', 'archive'))
+        + (
+            '::'.join((repository, info_arguments.archive))
+            if info_arguments.archive
+            else repository,
+        )
     )
 
     return execute_command(
-        full_command, output_log_level=None if info_arguments.json else logging.WARNING
+        full_command,
+        output_log_level=None if info_arguments.json else logging.WARNING,
+        borg_local_path=local_path,
     )

borgmatic/borg/init.py

@@ -1,7 +1,7 @@
 import logging
 import subprocess
 
-from borgmatic.execute import execute_command
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -11,6 +11,7 @@ INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
 
 def initialize_repository(
     repository,
+    storage_config,
     encryption_mode,
     append_only=None,
     storage_quota=None,
@@ -18,11 +19,17 @@ def initialize_repository(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a Borg encryption mode, whether the repository should
-    be append-only, and the storage quota to use, initialize the repository. If the repository
-    already exists, then log and skip initialization.
+    Given a local or remote repository path, a storage configuration dict, a Borg encryption mode,
+    whether the repository should be append-only, and the storage quota to use, initialize the
+    repository. If the repository already exists, then log and skip initialization.
     '''
-    info_command = (local_path, 'info', repository)
+    info_command = (
+        (local_path, 'info')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (repository,)
+    )
     logger.debug(' '.join(info_command))
 
     try:
@@ -33,15 +40,19 @@ def initialize_repository(
         if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
             raise
 
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('init', '')
+
     init_command = (
-        (local_path, 'init', repository)
+        (local_path, 'init')
         + (('--encryption', encryption_mode) if encryption_mode else ())
         + (('--append-only',) if append_only else ())
         + (('--storage-quota', storage_quota) if storage_quota else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--remote-path', remote_path) if remote_path else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (repository,)
     )
 
-    # Don't use execute_command() here because it doesn't support interactive prompts.
-    subprocess.check_call(init_command)
+    # Do not capture output here, so as to support interactive prompts.
+    execute_command(init_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)

borgmatic/borg/list.py

@@ -6,6 +6,47 @@ from borgmatic.execute import execute_command
 logger = logging.getLogger(__name__)
 
 
+# A hack to convince Borg to exclude archives ending in ".checkpoint". This assumes that a
+# non-checkpoint archive name ends in a digit (e.g. from a timestamp).
+BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[0123456789]'
+
+
+def resolve_archive_name(repository, archive, storage_config, local_path='borg', remote_path=None):
+    '''
+    Given a local or remote repository path, an archive name, a storage config dict, a local Borg
+    path, and a remote Borg path, simply return the archive name. But if the archive name is
+    "latest", then instead introspect the repository for the latest successful (non-checkpoint)
+    archive, and return its name.
+
+    Raise ValueError if "latest" is given but there are no archives in the repository.
+    '''
+    if archive != "latest":
+        return archive
+
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'list')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags('glob-archives', BORG_EXCLUDE_CHECKPOINTS_GLOB)
+        + make_flags('last', 1)
+        + ('--short', repository)
+    )
+
+    output = execute_command(full_command, output_log_level=None, borg_local_path=local_path)
+    try:
+        latest_archive = output.strip().splitlines()[-1]
+    except IndexError:
+        raise ValueError('No archives found in the repository')
+
+    logger.debug('{}: Latest archive is {}'.format(repository, latest_archive))
+
+    return latest_archive
+
+
 def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
     '''
     Given a local or remote repository path, a storage config dict, and the arguments to the list
@@ -13,15 +54,11 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
     if an archive name is given, listing the files in that archive.
     '''
     lock_wait = storage_config.get('lock_wait', None)
+    if list_arguments.successful:
+        list_arguments.glob_archives = BORG_EXCLUDE_CHECKPOINTS_GLOB
 
     full_command = (
-        (
-            local_path,
-            'list',
-            '::'.join((repository, list_arguments.archive))
-            if list_arguments.archive
-            else repository,
-        )
+        (local_path, 'list')
         + (
             ('--info',)
             if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json
@@ -34,9 +71,19 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
         )
         + make_flags('remote-path', remote_path)
         + make_flags('lock-wait', lock_wait)
-        + make_flags_from_arguments(list_arguments, excludes=('repository', 'archive'))
+        + make_flags_from_arguments(
+            list_arguments, excludes=('repository', 'archive', 'paths', 'successful')
+        )
+        + (
+            '::'.join((repository, list_arguments.archive))
+            if list_arguments.archive
+            else repository,
+        )
+        + (tuple(list_arguments.paths) if list_arguments.paths else ())
     )
 
     return execute_command(
-        full_command, output_log_level=None if list_arguments.json else logging.WARNING
+        full_command,
+        output_log_level=None if list_arguments.json else logging.WARNING,
+        borg_local_path=local_path,
     )

borgmatic/borg/mount.py

@@ -0,0 +1,46 @@
+import logging
+
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def mount_archive(
+    repository,
+    archive,
+    mount_point,
+    paths,
+    foreground,
+    options,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, an optional archive name, a filesystem mount point,
+    zero or more paths to mount from the archive, extra Borg mount options, a storage configuration
+    dict, and optional local and remote Borg paths, mount the archive onto the mount point.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'mount')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--foreground',) if foreground else ())
+        + (('-o', options) if options else ())
+        + (('::'.join((repository, archive)),) if archive else (repository,))
+        + (mount_point,)
+        + (tuple(paths) if paths else ())
+    )
+
+    # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
+    if foreground:
+        execute_command(full_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)
+        return
+
+    execute_command(full_command, borg_local_path=local_path)

borgmatic/borg/prune.py

@@ -41,6 +41,7 @@ def prune_archives(
     local_path='borg',
     remote_path=None,
     stats=False,
+    files=False,
 ):
     '''
     Given dry-run flag, a local or remote repository path, a storage config dict, and a
@@ -49,18 +50,26 @@ def prune_archives(
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('prune', '')
 
     full_command = (
-        (local_path, 'prune', repository)
+        (local_path, 'prune')
         + tuple(element for pair in _make_prune_flags(retention_config) for element in pair)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--stats',) if not dry_run and logger.isEnabledFor(logging.INFO) else ())
+        + (('--stats',) if stats and not dry_run else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--list',) if files else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
-        + (('--stats',) if stats else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (repository,)
     )
 
-    execute_command(full_command)
+    if (stats or files) and logger.getEffectiveLevel() == logging.WARNING:
+        output_log_level = logging.WARNING
+    else:
+        output_log_level = logging.INFO
+
+    execute_command(full_command, output_log_level=output_log_level, borg_local_path=local_path)

borgmatic/borg/umount.py

@@ -0,0 +1,20 @@
+import logging
+
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def unmount_archive(mount_point, local_path='borg'):
+    '''
+    Given a mounted filesystem mount point, and an optional local Borg paths, umount the filesystem
+    from the mount point.
+    '''
+    full_command = (
+        (local_path, 'umount')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (mount_point,)
+    )
+
+    execute_command(full_command)

borgmatic/commands/arguments.py

@@ -1,5 +1,5 @@
 import collections
-from argparse import ArgumentParser
+from argparse import Action, ArgumentParser
 
 from borgmatic.config import collect
 
@@ -9,19 +9,24 @@ SUBPARSER_ALIASES = {
     'create': ['--create', '-C'],
     'check': ['--check', '-k'],
     'extract': ['--extract', '-x'],
+    'export-tar': ['--export-tar'],
+    'mount': ['--mount', '-m'],
+    'umount': ['--umount', '-u'],
+    'restore': ['--restore', '-r'],
     'list': ['--list', '-l'],
     'info': ['--info', '-i'],
+    'borg': [],
 }
 
 
-def parse_subparser_arguments(unparsed_arguments, top_level_parser, subparsers):
+def parse_subparser_arguments(unparsed_arguments, subparsers):
     '''
-    Given a sequence of arguments, a top-level parser (containing subparsers), and a subparsers
-    object as returned by argparse.ArgumentParser().add_subparsers(), ask each subparser to parse
-    its own arguments and the top-level parser to parse any remaining arguments.
+    Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser
+    instance, give each requested action's subparser a shot at parsing all arguments. This allows
+    common arguments like "--repository" to be shared across multiple subparsers.
 
-    Return the result as a dict mapping from subparser name (or "global") to a parsed namespace of
-    arguments.
+    Return the result as a tuple of (a dict mapping from subparser name to a parsed namespace of
+    arguments, a list of remaining arguments not claimed by any subparser).
     '''
     arguments = collections.OrderedDict()
     remaining_arguments = list(unparsed_arguments)
@@ -31,33 +36,76 @@ def parse_subparser_arguments(unparsed_arguments, top_level_parser, subparsers):
         for alias in aliases
     }
 
-    # Give each requested action's subparser a shot at parsing all arguments.
-    for subparser_name, subparser in subparsers.choices.items():
-        if subparser_name not in unparsed_arguments:
+    # If the "borg" action is used, skip all other subparsers. This avoids confusion like
+    # "borg list" triggering borgmatic's own list action.
+    if 'borg' in unparsed_arguments:
+        subparsers = {'borg': subparsers['borg']}
+
+    for subparser_name, subparser in subparsers.items():
+        if subparser_name not in remaining_arguments:
             continue
 
-        remaining_arguments.remove(subparser_name)
         canonical_name = alias_to_subparser_name.get(subparser_name, subparser_name)
 
-        parsed, remaining = subparser.parse_known_args(unparsed_arguments)
+        # If a parsed value happens to be the same as the name of a subparser, remove it from the
+        # remaining arguments. This prevents, for instance, "check --only extract" from triggering
+        # the "extract" subparser.
+        parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+        for value in vars(parsed).values():
+            if isinstance(value, str):
+                if value in subparsers:
+                    remaining_arguments.remove(value)
+            elif isinstance(value, list):
+                for item in value:
+                    if item in subparsers:
+                        remaining_arguments.remove(item)
+
         arguments[canonical_name] = parsed
 
     # If no actions are explicitly requested, assume defaults: prune, create, and check.
     if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
         for subparser_name in ('prune', 'create', 'check'):
-            subparser = subparsers.choices[subparser_name]
-            parsed, remaining = subparser.parse_known_args(unparsed_arguments)
+            subparser = subparsers[subparser_name]
+            parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
             arguments[subparser_name] = parsed
 
-    # Then ask each subparser, one by one, to greedily consume arguments. Any arguments that remain
-    # are global arguments.
-    for subparser_name in arguments.keys():
-        subparser = subparsers.choices[subparser_name]
-        parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+    remaining_arguments = list(unparsed_arguments)
 
-    arguments['global'] = top_level_parser.parse_args(remaining_arguments)
+    # Now ask each subparser, one by one, to greedily consume arguments.
+    for subparser_name, subparser in subparsers.items():
+        if subparser_name not in arguments.keys():
+            continue
 
-    return arguments
+        subparser = subparsers[subparser_name]
+        unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the
+    # "borg" action.
+    if 'borg' in arguments:
+        borg_options_index = remaining_arguments.index('borg') + 1
+        arguments['borg'].options = remaining_arguments[borg_options_index:]
+        remaining_arguments = remaining_arguments[:borg_options_index]
+
+    # Remove the subparser names themselves.
+    for subparser_name, subparser in subparsers.items():
+        if subparser_name in remaining_arguments:
+            remaining_arguments.remove(subparser_name)
+
+    return (arguments, remaining_arguments)
+
+
+class Extend_action(Action):
+    '''
+    An argparse action to support Python 3.8's "extend" action in older versions of Python.
+    '''
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = getattr(namespace, self.dest, None)
+
+        if items:
+            items.extend(values)
+        else:
+            setattr(namespace, self.dest, list(values))
 
 
 def parse_arguments(*unparsed_arguments):
@@ -65,9 +113,11 @@ def parse_arguments(*unparsed_arguments):
     Given command-line arguments with which this script was invoked, parse the arguments and return
     them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance.
     '''
-    config_paths = collect.get_default_config_paths()
+    config_paths = collect.get_default_config_paths(expand_home=True)
+    unexpanded_config_paths = collect.get_default_config_paths(expand_home=False)
 
     global_parser = ArgumentParser(add_help=False)
+    global_parser.register('action', 'extend', Extend_action)
     global_group = global_parser.add_argument_group('global arguments')
 
     global_group.add_argument(
@@ -77,7 +127,7 @@ def parse_arguments(*unparsed_arguments):
         dest='config_paths',
         default=config_paths,
         help='Configuration filenames or directories, defaults to: {}'.format(
-            ' '.join(config_paths)
+            ' '.join(unexpanded_config_paths)
         ),
     )
     global_group.add_argument(
@@ -99,16 +149,44 @@ def parse_arguments(*unparsed_arguments):
         '-v',
         '--verbosity',
         type=int,
-        choices=range(0, 3),
+        choices=range(-1, 3),
         default=0,
-        help='Display verbose progress to the console (from none to lots: 0, 1, or 2)',
+        help='Display verbose progress to the console (from only errors to very verbose: -1, 0, 1, or 2)',
     )
     global_group.add_argument(
         '--syslog-verbosity',
         type=int,
-        choices=range(0, 3),
+        choices=range(-1, 3),
         default=0,
-        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2). Ignored when console is interactive',
+        help='Log verbose progress to syslog (from only errors to very verbose: -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given',
+    )
+    global_group.add_argument(
+        '--log-file-verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=0,
+        help='Log verbose progress to log file (from only errors to very verbose: -1, 0, 1, or 2). Only used when --log-file is given',
+    )
+    global_group.add_argument(
+        '--monitoring-verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=0,
+        help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)',
+    )
+    global_group.add_argument(
+        '--log-file',
+        type=str,
+        default=None,
+        help='Write log messages to this file instead of syslog',
+    )
+    global_group.add_argument(
+        '--override',
+        metavar='SECTION.OPTION=VALUE',
+        nargs='+',
+        dest='overrides',
+        action='extend',
+        help='One or more configuration file options to override with specified values',
     )
     global_group.add_argument(
         '--version',
@@ -120,9 +198,9 @@ def parse_arguments(*unparsed_arguments):
 
     top_level_parser = ArgumentParser(
         description='''
-            A simple wrapper script for the Borg backup software that creates and prunes backups.
-            If none of the action options are given, then borgmatic defaults to: prune, create, and
-            check archives.
+            Simple, configuration-driven backup software for servers and workstations. If none of
+            the action options are given, then borgmatic defaults to: prune, create, and check
+            archives.
             ''',
         parents=[global_parser],
     )
@@ -175,6 +253,9 @@ def parse_arguments(*unparsed_arguments):
         action='store_true',
         help='Display statistics of archive',
     )
+    prune_group.add_argument(
+        '--files', dest='files', default=False, action='store_true', help='Show per-file details'
+    )
     prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
     create_parser = subparsers.add_parser(
@@ -190,7 +271,7 @@ def parse_arguments(*unparsed_arguments):
         dest='progress',
         default=False,
         action='store_true',
-        help='Display progress for each file as it is processed',
+        help='Display progress for each file as it is backed up',
     )
     create_group.add_argument(
         '--stats',
@@ -199,6 +280,9 @@ def parse_arguments(*unparsed_arguments):
         action='store_true',
         help='Display statistics of archive',
     )
+    create_group.add_argument(
+        '--files', dest='files', default=False, action='store_true', help='Show per-file details'
+    )
     create_group.add_argument(
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )
@@ -212,12 +296,34 @@ def parse_arguments(*unparsed_arguments):
         add_help=False,
     )
     check_group = check_parser.add_argument_group('check arguments')
+    check_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is checked',
+    )
+    check_group.add_argument(
+        '--repair',
+        dest='repair',
+        default=False,
+        action='store_true',
+        help='Attempt to repair any inconsistencies found (experimental and only for interactive use)',
+    )
+    check_group.add_argument(
+        '--only',
+        metavar='CHECK',
+        choices=('repository', 'archives', 'data', 'extract'),
+        dest='only',
+        action='append',
+        help='Run a particular consistency check (repository, archives, data, or extract) instead of configured checks; can specify flag multiple times',
+    )
     check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
     extract_parser = subparsers.add_parser(
         'extract',
         aliases=SUBPARSER_ALIASES['extract'],
-        help='Extract a named archive to the current directory',
+        help='Extract files from a named archive to the current directory',
         description='Extract a named archive to the current directory',
         add_help=False,
     )
@@ -226,24 +332,167 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to extract, defaults to the configured repository if there is only one',
     )
-    extract_group.add_argument('--archive', help='Name of archive to operate on', required=True)
     extract_group.add_argument(
+        '--archive', help='Name of archive to extract (or "latest")', required=True
+    )
+    extract_group.add_argument(
+        '--path',
         '--restore-path',
+        metavar='PATH',
         nargs='+',
-        dest='restore_paths',
-        help='Paths to restore from archive, defaults to the entire archive',
+        dest='paths',
+        help='Paths to extract from archive, defaults to the entire archive',
+    )
+    extract_group.add_argument(
+        '--destination',
+        metavar='PATH',
+        dest='destination',
+        help='Directory to extract files into, defaults to the current directory',
+    )
+    extract_group.add_argument(
+        '--strip-components',
+        type=int,
+        metavar='NUMBER',
+        dest='strip_components',
+        help='Number of leading path components to remove from each extracted path. Skip paths with fewer elements',
     )
     extract_group.add_argument(
         '--progress',
         dest='progress',
         default=False,
         action='store_true',
-        help='Display progress for each file as it is processed',
+        help='Display progress for each file as it is extracted',
     )
     extract_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
+    export_tar_parser = subparsers.add_parser(
+        'export-tar',
+        aliases=SUBPARSER_ALIASES['export-tar'],
+        help='Export an archive to a tar-formatted file or stream',
+        description='Export an archive to a tar-formatted file or stream',
+        add_help=False,
+    )
+    export_tar_group = export_tar_parser.add_argument_group('export-tar arguments')
+    export_tar_group.add_argument(
+        '--repository',
+        help='Path of repository to export from, defaults to the configured repository if there is only one',
+    )
+    export_tar_group.add_argument(
+        '--archive', help='Name of archive to export (or "latest")', required=True
+    )
+    export_tar_group.add_argument(
+        '--path',
+        metavar='PATH',
+        nargs='+',
+        dest='paths',
+        help='Paths to export from archive, defaults to the entire archive',
+    )
+    export_tar_group.add_argument(
+        '--destination',
+        metavar='PATH',
+        dest='destination',
+        help='Path to destination export tar file, or "-" for stdout (but be careful about dirtying output with --verbosity or --files)',
+        required=True,
+    )
+    export_tar_group.add_argument(
+        '--tar-filter', help='Name of filter program to pipe data through'
+    )
+    export_tar_group.add_argument(
+        '--files', default=False, action='store_true', help='Show per-file details'
+    )
+    export_tar_group.add_argument(
+        '--strip-components',
+        type=int,
+        metavar='NUMBER',
+        dest='strip_components',
+        help='Number of leading path components to remove from each exported path. Skip paths with fewer elements',
+    )
+    export_tar_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    mount_parser = subparsers.add_parser(
+        'mount',
+        aliases=SUBPARSER_ALIASES['mount'],
+        help='Mount files from a named archive as a FUSE filesystem',
+        description='Mount a named archive as a FUSE filesystem',
+        add_help=False,
+    )
+    mount_group = mount_parser.add_argument_group('mount arguments')
+    mount_group.add_argument(
+        '--repository',
+        help='Path of repository to use, defaults to the configured repository if there is only one',
+    )
+    mount_group.add_argument('--archive', help='Name of archive to mount (or "latest")')
+    mount_group.add_argument(
+        '--mount-point',
+        metavar='PATH',
+        dest='mount_point',
+        help='Path where filesystem is to be mounted',
+        required=True,
+    )
+    mount_group.add_argument(
+        '--path',
+        metavar='PATH',
+        nargs='+',
+        dest='paths',
+        help='Paths to mount from archive, defaults to the entire archive',
+    )
+    mount_group.add_argument(
+        '--foreground',
+        dest='foreground',
+        default=False,
+        action='store_true',
+        help='Stay in foreground until ctrl-C is pressed',
+    )
+    mount_group.add_argument('--options', dest='options', help='Extra Borg mount options')
+    mount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    umount_parser = subparsers.add_parser(
+        'umount',
+        aliases=SUBPARSER_ALIASES['umount'],
+        help='Unmount a FUSE filesystem that was mounted with "borgmatic mount"',
+        description='Unmount a mounted FUSE filesystem',
+        add_help=False,
+    )
+    umount_group = umount_parser.add_argument_group('umount arguments')
+    umount_group.add_argument(
+        '--mount-point',
+        metavar='PATH',
+        dest='mount_point',
+        help='Path of filesystem to unmount',
+        required=True,
+    )
+    umount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    restore_parser = subparsers.add_parser(
+        'restore',
+        aliases=SUBPARSER_ALIASES['restore'],
+        help='Restore database dumps from a named archive',
+        description='Restore database dumps from a named archive. (To extract files instead, use "borgmatic extract".)',
+        add_help=False,
+    )
+    restore_group = restore_parser.add_argument_group('restore arguments')
+    restore_group.add_argument(
+        '--repository',
+        help='Path of repository to restore from, defaults to the configured repository if there is only one',
+    )
+    restore_group.add_argument(
+        '--archive', help='Name of archive to restore from (or "latest")', required=True
+    )
+    restore_group.add_argument(
+        '--database',
+        metavar='NAME',
+        nargs='+',
+        dest='databases',
+        help='Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
     list_parser = subparsers.add_parser(
         'list',
         aliases=SUBPARSER_ALIASES['list'],
@@ -253,10 +502,16 @@ def parse_arguments(*unparsed_arguments):
     )
     list_group = list_parser.add_argument_group('list arguments')
     list_group.add_argument(
-        '--repository',
-        help='Path of repository to list, defaults to the configured repository if there is only one',
+        '--repository', help='Path of repository to list, defaults to the configured repositories',
+    )
+    list_group.add_argument('--archive', help='Name of archive to list (or "latest")')
+    list_group.add_argument(
+        '--path',
+        metavar='PATH',
+        nargs='+',
+        dest='paths',
+        help='Paths to list from archive, defaults to the entire archive',
     )
-    list_group.add_argument('--archive', help='Name of archive to list')
     list_group.add_argument(
         '--short', default=False, action='store_true', help='Output only archive or path names'
     )
@@ -270,6 +525,12 @@ def parse_arguments(*unparsed_arguments):
     list_group.add_argument(
         '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob'
     )
+    list_group.add_argument(
+        '--successful',
+        default=False,
+        action='store_true',
+        help='Only list archive names of successful (non-checkpoint) backups',
+    )
     list_group.add_argument(
         '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
     )
@@ -277,7 +538,7 @@ def parse_arguments(*unparsed_arguments):
         '--first', metavar='N', help='List first N archives after other filters are applied'
     )
     list_group.add_argument(
-        '--last', metavar='N', help='List first N archives after other filters are applied'
+        '--last', metavar='N', help='List last N archives after other filters are applied'
     )
     list_group.add_argument(
         '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
@@ -287,7 +548,7 @@ def parse_arguments(*unparsed_arguments):
     )
     list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
     list_group.add_argument(
-        '--pattern-from',
+        '--patterns-from',
         metavar='FILENAME',
         help='Include or exclude paths matching patterns from pattern file, one per line',
     )
@@ -305,7 +566,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to show info for, defaults to the configured repository if there is only one',
     )
-    info_group.add_argument('--archive', help='Name of archive to show info for')
+    info_group.add_argument('--archive', help='Name of archive to show info for (or "latest")')
     info_group.add_argument(
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )
@@ -327,11 +588,36 @@ def parse_arguments(*unparsed_arguments):
         help='Show info for first N archives after other filters are applied',
     )
     info_group.add_argument(
-        '--last', metavar='N', help='Show info for first N archives after other filters are applied'
+        '--last', metavar='N', help='Show info for last N archives after other filters are applied'
     )
     info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    arguments = parse_subparser_arguments(unparsed_arguments, top_level_parser, subparsers)
+    borg_parser = subparsers.add_parser(
+        'borg',
+        aliases=SUBPARSER_ALIASES['borg'],
+        help='Run an arbitrary Borg command',
+        description='Run an arbitrary Borg command based on borgmatic\'s configuration',
+        add_help=False,
+    )
+    borg_group = borg_parser.add_argument_group('borg arguments')
+    borg_group.add_argument(
+        '--repository',
+        help='Path of repository to pass to Borg, defaults to the configured repositories',
+    )
+    borg_group.add_argument('--archive', help='Name of archive to pass to Borg (or "latest")')
+    borg_group.add_argument(
+        '--',
+        metavar='OPTION',
+        dest='options',
+        nargs='+',
+        help='Options to pass to Borg, command first ("create", "list", etc). "--" is optional. To specify the repository or the archive, you must use --repository or --archive instead of providing them here.',
+    )
+    borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    arguments, remaining_arguments = parse_subparser_arguments(
+        unparsed_arguments, subparsers.choices
+    )
+    arguments['global'] = top_level_parser.parse_args(remaining_arguments)
 
     if arguments['global'].excludes_filename:
         raise ValueError(
@@ -341,6 +627,9 @@ def parse_arguments(*unparsed_arguments):
     if 'init' in arguments and arguments['global'].dry_run:
         raise ValueError('The init action cannot be used with the --dry-run option')
 
+    if 'list' in arguments and arguments['list'].glob_archives and arguments['list'].successful:
+        raise ValueError('The --glob-archives and --successful options cannot be used together')
+
     if (
         'list' in arguments
         and 'info' in arguments

borgmatic/commands/borgmatic.py

@@ -1,4 +1,5 @@
 import collections
+import copy
 import json
 import logging
 import os
@@ -8,17 +9,21 @@ from subprocess import CalledProcessError
 import colorama
 import pkg_resources
 
-from borgmatic import hook
+from borgmatic.borg import borg as borg_borg
 from borgmatic.borg import check as borg_check
 from borgmatic.borg import create as borg_create
 from borgmatic.borg import environment as borg_environment
+from borgmatic.borg import export_tar as borg_export_tar
 from borgmatic.borg import extract as borg_extract
 from borgmatic.borg import info as borg_info
 from borgmatic.borg import init as borg_init
 from borgmatic.borg import list as borg_list
+from borgmatic.borg import mount as borg_mount
 from borgmatic.borg import prune as borg_prune
+from borgmatic.borg import umount as borg_umount
 from borgmatic.commands.arguments import parse_arguments
 from borgmatic.config import checks, collect, convert, validate
+from borgmatic.hooks import command, dispatch, dump, monitor
 from borgmatic.logger import configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
@@ -28,13 +33,16 @@ logger = logging.getLogger(__name__)
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
 
 
-def run_configuration(config_filename, config, arguments):  # pragma: no cover
+def run_configuration(config_filename, config, arguments):
     '''
     Given a config filename, the corresponding parsed config dict, and command-line arguments as a
     dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
     backups, consistency checks, and/or other actions.
 
-    Yield JSON output strings from executing any actions that produce JSON.
+    Yield a combination of:
+
+      * JSON output strings from successfully executing any actions that produce JSON
+      * logging.LogRecord instances containing errors from any actions or backup hooks that fail
     '''
     (location, storage, retention, consistency, hooks) = (
         config.get(section_name, {})
@@ -42,49 +50,201 @@ def run_configuration(config_filename, config, arguments):  # pragma: no cover
     )
     global_arguments = arguments['global']
 
-    try:
-        local_path = location.get('local_path', 'borg')
-        remote_path = location.get('remote_path')
-        borg_environment.initialize(storage)
+    local_path = location.get('local_path', 'borg')
+    remote_path = location.get('remote_path')
+    borg_environment.initialize(storage)
+    encountered_error = None
+    error_repository = ''
+    prune_create_or_check = {'prune', 'create', 'check'}.intersection(arguments)
+    monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity)
 
+    try:
+        if prune_create_or_check:
+            dispatch.call_hooks(
+                'initialize_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
+        if 'prune' in arguments:
+            command.execute_hook(
+                hooks.get('before_prune'),
+                hooks.get('umask'),
+                config_filename,
+                'pre-prune',
+                global_arguments.dry_run,
+            )
         if 'create' in arguments:
-            hook.execute_hook(
+            command.execute_hook(
                 hooks.get('before_backup'),
                 hooks.get('umask'),
                 config_filename,
                 'pre-backup',
                 global_arguments.dry_run,
             )
-
-        for repository_path in location['repositories']:
-            yield from run_actions(
-                arguments=arguments,
-                location=location,
-                storage=storage,
-                retention=retention,
-                consistency=consistency,
-                local_path=local_path,
-                remote_path=remote_path,
-                repository_path=repository_path,
-            )
-
-        if 'create' in arguments:
-            hook.execute_hook(
-                hooks.get('after_backup'),
+        if 'check' in arguments:
+            command.execute_hook(
+                hooks.get('before_check'),
                 hooks.get('umask'),
                 config_filename,
-                'post-backup',
+                'pre-check',
                 global_arguments.dry_run,
             )
-    except (OSError, CalledProcessError):
-        hook.execute_hook(
-            hooks.get('on_error'),
-            hooks.get('umask'),
-            config_filename,
-            'on-error',
-            global_arguments.dry_run,
+        if 'extract' in arguments:
+            command.execute_hook(
+                hooks.get('before_extract'),
+                hooks.get('umask'),
+                config_filename,
+                'pre-extract',
+                global_arguments.dry_run,
+            )
+        if prune_create_or_check:
+            dispatch.call_hooks(
+                'ping_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitor.State.START,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
+    except (OSError, CalledProcessError) as error:
+        if command.considered_soft_failure(config_filename, error):
+            return
+
+        encountered_error = error
+        yield from make_error_log_records(
+            '{}: Error running pre hook'.format(config_filename), error
         )
-        raise
+
+    if not encountered_error:
+        for repository_path in location['repositories']:
+            try:
+                yield from run_actions(
+                    arguments=arguments,
+                    location=location,
+                    storage=storage,
+                    retention=retention,
+                    consistency=consistency,
+                    hooks=hooks,
+                    local_path=local_path,
+                    remote_path=remote_path,
+                    repository_path=repository_path,
+                )
+            except (OSError, CalledProcessError, ValueError) as error:
+                encountered_error = error
+                error_repository = repository_path
+                yield from make_error_log_records(
+                    '{}: Error running actions for repository'.format(repository_path), error
+                )
+
+    if not encountered_error:
+        try:
+            if 'prune' in arguments:
+                command.execute_hook(
+                    hooks.get('after_prune'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'post-prune',
+                    global_arguments.dry_run,
+                )
+            if 'create' in arguments:
+                dispatch.call_hooks(
+                    'remove_database_dumps',
+                    hooks,
+                    config_filename,
+                    dump.DATABASE_HOOK_NAMES,
+                    location,
+                    global_arguments.dry_run,
+                )
+                command.execute_hook(
+                    hooks.get('after_backup'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'post-backup',
+                    global_arguments.dry_run,
+                )
+            if 'check' in arguments:
+                command.execute_hook(
+                    hooks.get('after_check'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'post-check',
+                    global_arguments.dry_run,
+                )
+            if 'extract' in arguments:
+                command.execute_hook(
+                    hooks.get('after_extract'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'post-extract',
+                    global_arguments.dry_run,
+                )
+            if prune_create_or_check:
+                dispatch.call_hooks(
+                    'ping_monitor',
+                    hooks,
+                    config_filename,
+                    monitor.MONITOR_HOOK_NAMES,
+                    monitor.State.FINISH,
+                    monitoring_log_level,
+                    global_arguments.dry_run,
+                )
+                dispatch.call_hooks(
+                    'destroy_monitor',
+                    hooks,
+                    config_filename,
+                    monitor.MONITOR_HOOK_NAMES,
+                    monitoring_log_level,
+                    global_arguments.dry_run,
+                )
+        except (OSError, CalledProcessError) as error:
+            if command.considered_soft_failure(config_filename, error):
+                return
+
+            encountered_error = error
+            yield from make_error_log_records(
+                '{}: Error running post hook'.format(config_filename), error
+            )
+
+    if encountered_error and prune_create_or_check:
+        try:
+            command.execute_hook(
+                hooks.get('on_error'),
+                hooks.get('umask'),
+                config_filename,
+                'on-error',
+                global_arguments.dry_run,
+                repository=error_repository,
+                error=encountered_error,
+                output=getattr(encountered_error, 'output', ''),
+            )
+            dispatch.call_hooks(
+                'ping_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitor.State.FAIL,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
+            dispatch.call_hooks(
+                'destroy_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
+        except (OSError, CalledProcessError) as error:
+            if command.considered_soft_failure(config_filename, error):
+                return
+
+            yield from make_error_log_records(
+                '{}: Error running on-error hook'.format(config_filename), error
+            )
 
 
 def run_actions(
@@ -94,6 +254,7 @@ def run_actions(
     storage,
     retention,
     consistency,
+    hooks,
     local_path,
     remote_path,
     repository_path
@@ -104,6 +265,9 @@ def run_actions(
     from the command-line arguments on the given repository.
 
     Yield JSON output strings from executing any actions that produce JSON.
+
+    Raise OSError or subprocess.CalledProcessError if an error occurs running a command for an
+    action. Raise ValueError if the arguments or configuration passed to action are invalid.
     '''
     repository = os.path.expanduser(repository_path)
     global_arguments = arguments['global']
@@ -112,6 +276,7 @@ def run_actions(
         logger.info('{}: Initializing repository'.format(repository))
         borg_init.initialize_repository(
             repository,
+            storage,
             arguments['init'].encryption_mode,
             arguments['init'].append_only,
             arguments['init'].storage_quota,
@@ -128,9 +293,28 @@ def run_actions(
             local_path=local_path,
             remote_path=remote_path,
             stats=arguments['prune'].stats,
+            files=arguments['prune'].files,
         )
     if 'create' in arguments:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
+        dispatch.call_hooks(
+            'remove_database_dumps',
+            hooks,
+            repository,
+            dump.DATABASE_HOOK_NAMES,
+            location,
+            global_arguments.dry_run,
+        )
+        active_dumps = dispatch.call_hooks(
+            'dump_databases',
+            hooks,
+            repository,
+            dump.DATABASE_HOOK_NAMES,
+            location,
+            global_arguments.dry_run,
+        )
+        stream_processes = [process for processes in active_dumps.values() for process in processes]
+
         json_output = borg_create.create_archive(
             global_arguments.dry_run,
             repository,
@@ -141,57 +325,244 @@ def run_actions(
             progress=arguments['create'].progress,
             stats=arguments['create'].stats,
             json=arguments['create'].json,
+            files=arguments['create'].files,
+            stream_processes=stream_processes,
         )
         if json_output:
             yield json.loads(json_output)
+
     if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
-            repository, storage, consistency, local_path=local_path, remote_path=remote_path
+            repository,
+            storage,
+            consistency,
+            local_path=local_path,
+            remote_path=remote_path,
+            progress=arguments['check'].progress,
+            repair=arguments['check'].repair,
+            only_checks=arguments['check'].only,
         )
     if 'extract' in arguments:
-        if arguments['extract'].repository is None or repository == arguments['extract'].repository:
+        if arguments['extract'].repository is None or validate.repositories_match(
+            repository, arguments['extract'].repository
+        ):
             logger.info(
                 '{}: Extracting archive {}'.format(repository, arguments['extract'].archive)
             )
             borg_extract.extract_archive(
                 global_arguments.dry_run,
                 repository,
-                arguments['extract'].archive,
-                arguments['extract'].restore_paths,
+                borg_list.resolve_archive_name(
+                    repository, arguments['extract'].archive, storage, local_path, remote_path
+                ),
+                arguments['extract'].paths,
                 location,
                 storage,
                 local_path=local_path,
                 remote_path=remote_path,
+                destination_path=arguments['extract'].destination,
+                strip_components=arguments['extract'].strip_components,
                 progress=arguments['extract'].progress,
             )
+    if 'export-tar' in arguments:
+        if arguments['export-tar'].repository is None or validate.repositories_match(
+            repository, arguments['export-tar'].repository
+        ):
+            logger.info(
+                '{}: Exporting archive {} as tar file'.format(
+                    repository, arguments['export-tar'].archive
+                )
+            )
+            borg_export_tar.export_tar_archive(
+                global_arguments.dry_run,
+                repository,
+                borg_list.resolve_archive_name(
+                    repository, arguments['export-tar'].archive, storage, local_path, remote_path
+                ),
+                arguments['export-tar'].paths,
+                arguments['export-tar'].destination,
+                storage,
+                local_path=local_path,
+                remote_path=remote_path,
+                tar_filter=arguments['export-tar'].tar_filter,
+                files=arguments['export-tar'].files,
+                strip_components=arguments['export-tar'].strip_components,
+            )
+    if 'mount' in arguments:
+        if arguments['mount'].repository is None or validate.repositories_match(
+            repository, arguments['mount'].repository
+        ):
+            if arguments['mount'].archive:
+                logger.info(
+                    '{}: Mounting archive {}'.format(repository, arguments['mount'].archive)
+                )
+            else:
+                logger.info('{}: Mounting repository'.format(repository))
+
+            borg_mount.mount_archive(
+                repository,
+                borg_list.resolve_archive_name(
+                    repository, arguments['mount'].archive, storage, local_path, remote_path
+                ),
+                arguments['mount'].mount_point,
+                arguments['mount'].paths,
+                arguments['mount'].foreground,
+                arguments['mount'].options,
+                storage,
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+    if 'restore' in arguments:
+        if arguments['restore'].repository is None or validate.repositories_match(
+            repository, arguments['restore'].repository
+        ):
+            logger.info(
+                '{}: Restoring databases from archive {}'.format(
+                    repository, arguments['restore'].archive
+                )
+            )
+            dispatch.call_hooks(
+                'remove_database_dumps',
+                hooks,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                location,
+                global_arguments.dry_run,
+            )
+
+            restore_names = arguments['restore'].databases or []
+            if 'all' in restore_names:
+                restore_names = []
+
+            archive_name = borg_list.resolve_archive_name(
+                repository, arguments['restore'].archive, storage, local_path, remote_path
+            )
+            found_names = set()
+
+            for hook_name, per_hook_restore_databases in hooks.items():
+                if hook_name not in dump.DATABASE_HOOK_NAMES:
+                    continue
+
+                for restore_database in per_hook_restore_databases:
+                    database_name = restore_database['name']
+                    if restore_names and database_name not in restore_names:
+                        continue
+
+                    found_names.add(database_name)
+                    dump_pattern = dispatch.call_hooks(
+                        'make_database_dump_pattern',
+                        hooks,
+                        repository,
+                        dump.DATABASE_HOOK_NAMES,
+                        location,
+                        database_name,
+                    )[hook_name]
+
+                    # Kick off a single database extract to stdout.
+                    extract_process = borg_extract.extract_archive(
+                        dry_run=global_arguments.dry_run,
+                        repository=repository,
+                        archive=archive_name,
+                        paths=dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
+                        location_config=location,
+                        storage_config=storage,
+                        local_path=local_path,
+                        remote_path=remote_path,
+                        destination_path='/',
+                        # A directory format dump isn't a single file, and therefore can't extract
+                        # to stdout. In this case, the extract_process return value is None.
+                        extract_to_stdout=bool(restore_database.get('format') != 'directory'),
+                    )
+
+                    # Run a single database restore, consuming the extract stdout (if any).
+                    dispatch.call_hooks(
+                        'restore_database_dump',
+                        {hook_name: [restore_database]},
+                        repository,
+                        dump.DATABASE_HOOK_NAMES,
+                        location,
+                        global_arguments.dry_run,
+                        extract_process,
+                    )
+
+            dispatch.call_hooks(
+                'remove_database_dumps',
+                hooks,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                location,
+                global_arguments.dry_run,
+            )
+
+            if not restore_names and not found_names:
+                raise ValueError('No databases were found to restore')
+
+            missing_names = sorted(set(restore_names) - found_names)
+            if missing_names:
+                raise ValueError(
+                    'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
+                        ', '.join(missing_names)
+                    )
+                )
+
     if 'list' in arguments:
-        if arguments['list'].repository is None or repository == arguments['list'].repository:
-            logger.info('{}: Listing archives'.format(repository))
+        if arguments['list'].repository is None or validate.repositories_match(
+            repository, arguments['list'].repository
+        ):
+            list_arguments = copy.copy(arguments['list'])
+            if not list_arguments.json:
+                logger.warning('{}: Listing archives'.format(repository))
+            list_arguments.archive = borg_list.resolve_archive_name(
+                repository, list_arguments.archive, storage, local_path, remote_path
+            )
             json_output = borg_list.list_archives(
                 repository,
                 storage,
-                list_arguments=arguments['list'],
+                list_arguments=list_arguments,
                 local_path=local_path,
                 remote_path=remote_path,
             )
             if json_output:
                 yield json.loads(json_output)
     if 'info' in arguments:
-        if arguments['info'].repository is None or repository == arguments['info'].repository:
-            logger.info('{}: Displaying summary info for archives'.format(repository))
+        if arguments['info'].repository is None or validate.repositories_match(
+            repository, arguments['info'].repository
+        ):
+            info_arguments = copy.copy(arguments['info'])
+            if not info_arguments.json:
+                logger.warning('{}: Displaying summary info for archives'.format(repository))
+            info_arguments.archive = borg_list.resolve_archive_name(
+                repository, info_arguments.archive, storage, local_path, remote_path
+            )
             json_output = borg_info.display_archives_info(
                 repository,
                 storage,
-                info_arguments=arguments['info'],
+                info_arguments=info_arguments,
                 local_path=local_path,
                 remote_path=remote_path,
             )
             if json_output:
                 yield json.loads(json_output)
+    if 'borg' in arguments:
+        if arguments['borg'].repository is None or validate.repositories_match(
+            repository, arguments['borg'].repository
+        ):
+            logger.warning('{}: Running arbitrary Borg command'.format(repository))
+            archive_name = borg_list.resolve_archive_name(
+                repository, arguments['borg'].archive, storage, local_path, remote_path
+            )
+            borg_borg.run_arbitrary_borg(
+                repository,
+                storage,
+                options=arguments['borg'].options,
+                archive=archive_name,
+                local_path=local_path,
+                remote_path=remote_path,
+            )
 
 
-def load_configurations(config_filenames):
+def load_configurations(config_filenames, overrides=None):
     '''
     Given a sequence of configuration filenames, load and validate each configuration file. Return
     the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
@@ -205,7 +576,7 @@ def load_configurations(config_filenames):
     for config_filename in config_filenames:
         try:
             configs[config_filename] = validate.parse_configuration(
-                config_filename, validate.schema_filename()
+                config_filename, validate.schema_filename(), overrides
             )
         except (ValueError, OSError, validate.Validation_error) as error:
             logs.extend(
@@ -226,6 +597,55 @@ def load_configurations(config_filenames):
     return (configs, logs)
 
 
+def log_record(suppress_log=False, **kwargs):
+    '''
+    Create a log record based on the given makeLogRecord() arguments, one of which must be
+    named "levelno". Log the record (unless suppress log is set) and return it.
+    '''
+    record = logging.makeLogRecord(kwargs)
+    if suppress_log:
+        return record
+
+    logger.handle(record)
+    return record
+
+
+def make_error_log_records(message, error=None):
+    '''
+    Given error message text and an optional exception object, yield a series of logging.LogRecord
+    instances with error summary information. As a side effect, log each record.
+    '''
+    if not error:
+        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        return
+
+    try:
+        raise error
+    except CalledProcessError as error:
+        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        if error.output:
+            # Suppress these logs for now and save full error output for the log summary at the end.
+            yield log_record(
+                levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output, suppress_log=True
+            )
+        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+    except (ValueError, OSError) as error:
+        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+    except:  # noqa: E722
+        # Raising above only as a means of determining the error type. Swallow the exception here
+        # because we don't want the exception to propagate out of this function.
+        pass
+
+
+def get_local_path(configs):
+    '''
+    Arbitrarily return the local path from the first configuration dict. Default to "borg" if not
+    set.
+    '''
+    return next(iter(configs.values())).get('location', {}).get('local_path', 'borg')
+
+
 def collect_configuration_run_summary_logs(configs, arguments):
     '''
     Given a dict of configuration filename to corresponding parsed configuration, and parsed
@@ -241,6 +661,8 @@ def collect_configuration_run_summary_logs(configs, arguments):
         repository = arguments['extract'].repository
     elif 'list' in arguments and arguments['list'].archive:
         repository = arguments['list'].repository
+    elif 'mount' in arguments:
+        repository = arguments['mount'].repository
     else:
         repository = None
 
@@ -248,16 +670,44 @@ def collect_configuration_run_summary_logs(configs, arguments):
         try:
             validate.guard_configuration_contains_repository(repository, configs)
         except ValueError as error:
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+            yield from make_error_log_records(str(error))
+            return
+
+    if not configs:
+        yield from make_error_log_records(
+            '{}: No valid configuration files found'.format(
+                ' '.join(arguments['global'].config_paths)
             )
+        )
+        return
+
+    if 'create' in arguments:
+        try:
+            for config_filename, config in configs.items():
+                hooks = config.get('hooks', {})
+                command.execute_hook(
+                    hooks.get('before_everything'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'pre-everything',
+                    arguments['global'].dry_run,
+                )
+        except (CalledProcessError, ValueError, OSError) as error:
+            yield from make_error_log_records('Error running pre-everything hook', error)
             return
 
     # Execute the actions corresponding to each configuration file.
     json_results = []
     for config_filename, config in configs.items():
-        try:
-            json_results.extend(list(run_configuration(config_filename, config, arguments)))
+        results = list(run_configuration(config_filename, config, arguments))
+        error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord))
+
+        if error_logs:
+            yield from make_error_log_records(
+                '{}: Error running configuration file'.format(config_filename)
+            )
+            yield from error_logs
+        else:
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.INFO,
@@ -265,45 +715,34 @@ def collect_configuration_run_summary_logs(configs, arguments):
                     msg='{}: Successfully ran configuration file'.format(config_filename),
                 )
             )
-        except CalledProcessError as error:
-            yield logging.makeLogRecord(
-                dict(
-                    levelno=logging.CRITICAL,
-                    levelname='CRITICAL',
-                    msg='{}: Error running configuration file'.format(config_filename),
-                )
-            )
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
-            )
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
-            )
-        except (ValueError, OSError) as error:
-            yield logging.makeLogRecord(
-                dict(
-                    levelno=logging.CRITICAL,
-                    levelname='CRITICAL',
-                    msg='{}: Error running configuration file'.format(config_filename),
-                )
-            )
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+            if results:
+                json_results.extend(results)
+
+    if 'umount' in arguments:
+        logger.info('Unmounting mount point {}'.format(arguments['umount'].mount_point))
+        try:
+            borg_umount.unmount_archive(
+                mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs)
             )
+        except (CalledProcessError, OSError) as error:
+            yield from make_error_log_records('Error unmounting mount point', error)
 
     if json_results:
         sys.stdout.write(json.dumps(json_results))
 
-    if not configs:
-        yield logging.makeLogRecord(
-            dict(
-                levelno=logging.CRITICAL,
-                levelname='CRITICAL',
-                msg='{}: No configuration files found'.format(
-                    ' '.join(arguments['global'].config_paths)
-                ),
-            )
-        )
+    if 'create' in arguments:
+        try:
+            for config_filename, config in configs.items():
+                hooks = config.get('hooks', {})
+                command.execute_hook(
+                    hooks.get('after_everything'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'post-everything',
+                    arguments['global'].dry_run,
+                )
+        except (CalledProcessError, ValueError, OSError) as error:
+            yield from make_error_log_records('Error running post-everything hook', error)
 
 
 def exit_with_help_link():  # pragma: no cover
@@ -337,26 +776,43 @@ def main():  # pragma: no cover
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
-    configs, parse_logs = load_configurations(config_filenames)
+    configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
 
-    colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
-    configure_logging(
-        verbosity_to_log_level(global_arguments.verbosity),
-        verbosity_to_log_level(global_arguments.syslog_verbosity),
+    any_json_flags = any(
+        getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()
     )
+    colorama.init(
+        autoreset=True,
+        strip=not should_do_markup(global_arguments.no_color or any_json_flags, configs),
+    )
+    try:
+        configure_logging(
+            verbosity_to_log_level(global_arguments.verbosity),
+            verbosity_to_log_level(global_arguments.syslog_verbosity),
+            verbosity_to_log_level(global_arguments.log_file_verbosity),
+            verbosity_to_log_level(global_arguments.monitoring_verbosity),
+            global_arguments.log_file,
+        )
+    except (FileNotFoundError, PermissionError) as error:
+        configure_logging(logging.CRITICAL)
+        logger.critical('Error configuring logging: {}'.format(error))
+        exit_with_help_link()
 
     logger.debug('Ensuring legacy configuration is upgraded')
     convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
 
-    summary_logs = list(collect_configuration_run_summary_logs(configs, arguments))
+    summary_logs = parse_logs + list(collect_configuration_run_summary_logs(configs, arguments))
+    summary_logs_max_level = max(log.levelno for log in summary_logs)
 
-    logger.info('')
-    logger.info('summary:')
-    [
+    for message in ('', 'summary:'):
+        log_record(
+            levelno=summary_logs_max_level,
+            levelname=logging.getLevelName(summary_logs_max_level),
+            msg=message,
+        )
+
+    for log in summary_logs:
         logger.handle(log)
-        for log in parse_logs + summary_logs
-        if log.levelno >= logger.getEffectiveLevel()
-    ]
 
-    if any(log.levelno == logging.CRITICAL for log in summary_logs):
+    if summary_logs_max_level >= logging.CRITICAL:
         exit_with_help_link()

borgmatic/commands/convert_config.py

@@ -99,7 +99,9 @@ def main():  # pragma: no cover
         )
 
         generate.write_configuration(
-            args.destination_config_filename, destination_config, mode=source_config_file_mode
+            args.destination_config_filename,
+            generate.render_configuration(destination_config),
+            mode=source_config_file_mode,
         )
 
         display_result(args)

borgmatic/commands/generate_config.py

@@ -12,12 +12,18 @@ def parse_arguments(*arguments):
     them as an ArgumentParser instance.
     '''
     parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
+    parser.add_argument(
+        '-s',
+        '--source',
+        dest='source_filename',
+        help='Optional YAML configuration file to merge into the generated configuration, useful for upgrading your configuration',
+    )
     parser.add_argument(
         '-d',
         '--destination',
         dest='destination_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help='Destination YAML configuration filename. Default: {}'.format(
+        help='Destination YAML configuration file. Default: {}'.format(
             DEFAULT_DESTINATION_CONFIG_FILENAME
         ),
     )
@@ -30,11 +36,21 @@ def main():  # pragma: no cover
         args = parse_arguments(*sys.argv[1:])
 
         generate.generate_sample_configuration(
-            args.destination_filename, validate.schema_filename()
+            args.source_filename, args.destination_filename, validate.schema_filename()
         )
 
         print('Generated a sample configuration file at {}.'.format(args.destination_filename))
         print()
+        if args.source_filename:
+            print(
+                'Merged in the contents of configuration file at {}.'.format(args.source_filename)
+            )
+            print('To review the changes made, run:')
+            print()
+            print(
+                '    diff --unified {} {}'.format(args.source_filename, args.destination_filename)
+            )
+            print()
         print('Please edit the file to suit your needs. The values are representative.')
         print('All fields are optional except where indicated.')
         print()

borgmatic/config/collect.py

@@ -1,20 +1,23 @@
 import os
 
 
-def get_default_config_paths():
+def get_default_config_paths(expand_home=True):
     '''
     Based on the value of the XDG_CONFIG_HOME and HOME environment variables, return a list of
     default configuration paths. This includes both system-wide configuration and configuration in
     the current user's home directory.
+
+    Don't expand the home directory ($HOME) if the expand home flag is False.
     '''
-    user_config_directory = os.getenv('XDG_CONFIG_HOME') or os.path.expandvars(
-        os.path.join('$HOME', '.config')
-    )
+    user_config_directory = os.getenv('XDG_CONFIG_HOME') or os.path.join('$HOME', '.config')
+    if expand_home:
+        user_config_directory = os.path.expandvars(user_config_directory)
 
     return [
         '/etc/borgmatic/config.yaml',
         '/etc/borgmatic.d',
         '%s/borgmatic/config.yaml' % user_config_directory,
+        '%s/borgmatic.d' % user_config_directory,
     ]
 
 
@@ -41,6 +44,9 @@ def collect_config_filenames(config_paths):
             yield path
             continue
 
+        if not os.access(path, os.R_OK):
+            continue
+
         for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
             matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')

borgmatic/config/convert.py

@@ -17,7 +17,7 @@ def _convert_section(source_section_config, section_schema):
             (
                 option_name,
                 int(option_value)
-                if section_schema['map'].get(option_name, {}).get('type') == 'int'
+                if section_schema['properties'].get(option_name, {}).get('type') == 'integer'
                 else option_value,
             )
             for option_name, option_value in source_section_config.items()
@@ -38,7 +38,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
     '''
     destination_config = yaml.comments.CommentedMap(
         [
-            (section_name, _convert_section(section_config, schema['map'][section_name]))
+            (section_name, _convert_section(section_config, schema['properties'][section_name]))
             for section_name, section_config in source_config._asdict().items()
         ]
     )
@@ -54,11 +54,11 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
 
     # Add comments to each section, and then add comments to the fields in each section.
-    generate.add_comments_to_configuration(destination_config, schema)
+    generate.add_comments_to_configuration_object(destination_config, schema)
 
     for section_name, section_config in destination_config.items():
-        generate.add_comments_to_configuration(
-            section_config, schema['map'][section_name], indent=generate.INDENT
+        generate.add_comments_to_configuration_object(
+            section_config, schema['properties'][section_name], indent=generate.INDENT
         )
 
     return destination_config

borgmatic/config/generate.py

@@ -1,8 +1,14 @@
+import collections
+import io
 import os
+import re
 
 from ruamel import yaml
 
+from borgmatic.config import load
+
 INDENT = 4
+SEQUENCE_INDENT = 2
 
 
 def _insert_newline_before_comment(config, field_name):
@@ -15,23 +21,34 @@ def _insert_newline_before_comment(config, field_name):
     )
 
 
-def _schema_to_sample_configuration(schema, level=0):
+def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     '''
     Given a loaded configuration schema, generate and return sample config for it. Include comments
-    for each section based on the schema "desc" description.
+    for each section based on the schema "description".
     '''
+    schema_type = schema.get('type')
     example = schema.get('example')
     if example is not None:
         return example
 
-    config = yaml.comments.CommentedMap(
-        [
-            (section_name, _schema_to_sample_configuration(section_schema, level + 1))
-            for section_name, section_schema in schema['map'].items()
-        ]
-    )
-
-    add_comments_to_configuration(config, schema, indent=(level * INDENT))
+    if schema_type == 'array':
+        config = yaml.comments.CommentedSeq(
+            [_schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
+        )
+        add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
+    elif schema_type == 'object':
+        config = yaml.comments.CommentedMap(
+            [
+                (field_name, _schema_to_sample_configuration(sub_schema, level + 1))
+                for field_name, sub_schema in schema['properties'].items()
+            ]
+        )
+        indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
+        add_comments_to_configuration_object(
+            config, schema, indent=indent, skip_first=parent_is_sequence
+        )
+    else:
+        raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema))
 
     return config
 
@@ -42,55 +59,54 @@ def _comment_out_line(line):
     if not stripped_line or stripped_line.startswith('#'):
         return line
 
-    # Comment out the names of optional sections.
-    one_indent = ' ' * INDENT
-    if not line.startswith(one_indent):
-        return '#' + line
+    # Comment out the names of optional sections, inserting the '#' after any indent for aesthetics.
+    matches = re.match(r'(\s*)', line)
+    indent_spaces = matches.group(0) if matches else ''
+    count_indent_spaces = len(indent_spaces)
 
-    # Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics.
-    return '#'.join((one_indent, line[INDENT:]))
-
-
-REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
-REQUIRED_SECTION_NAMES = {'location', 'retention'}
+    return '# '.join((indent_spaces, line[count_indent_spaces:]))
 
 
 def _comment_out_optional_configuration(rendered_config):
     '''
-    Post-process a rendered configuration string to comment out optional key/values. The idea is
-    that this prevents the user from having to comment out a bunch of configuration they don't care
-    about to get to a minimal viable configuration file.
+    Post-process a rendered configuration string to comment out optional key/values, as determined
+    by a sentinel in the comment before each key.
 
-    Ideally ruamel.yaml would support this during configuration generation, but it's not terribly
-    easy to accomplish that way.
+    The idea is that the pre-commented configuration prevents the user from having to comment out a
+    bunch of configuration they don't care about to get to a minimal viable configuration file.
+
+    Ideally ruamel.yaml would support commenting out keys during configuration generation, but it's
+    not terribly easy to accomplish that way.
     '''
     lines = []
-    required = False
+    optional = False
 
     for line in rendered_config.split('\n'):
-        key = line.strip().split(':')[0]
-
-        if key in REQUIRED_SECTION_NAMES:
-            lines.append(line)
+        # Upon encountering an optional configuration option, comment out lines until the next blank
+        # line.
+        if line.strip().startswith('# {}'.format(COMMENTED_OUT_SENTINEL)):
+            optional = True
             continue
 
-        # Upon encountering a required configuration option, skip commenting out lines until the
-        # next blank line.
-        if key in REQUIRED_KEYS:
-            required = True
-        elif not key:
-            required = False
+        # Hit a blank line, so reset commenting.
+        if not line.strip():
+            optional = False
 
-        lines.append(_comment_out_line(line) if not required else line)
+        lines.append(_comment_out_line(line) if optional else line)
 
     return '\n'.join(lines)
 
 
-def _render_configuration(config):
+def render_configuration(config):
     '''
     Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
     '''
-    return yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)
+    dumper = yaml.YAML()
+    dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
+    rendered = io.StringIO()
+    dumper.dump(config, rendered)
+
+    return rendered.getvalue()
 
 
 def write_configuration(config_filename, rendered_config, mode=0o600):
@@ -112,33 +128,159 @@ def write_configuration(config_filename, rendered_config, mode=0o600):
     os.chmod(config_filename, mode)
 
 
-def add_comments_to_configuration(config, schema, indent=0):
+def add_comments_to_configuration_sequence(config, schema, indent=0):
     '''
-    Using descriptions from a schema as a source, add those descriptions as comments to the given
-    config before each field. This function only adds comments for the top-most config map level.
-    Indent the comment the given number of characters.
+    If the given config sequence's items are object, then mine the schema for the description of the
+    object's first item, and slap that atop the sequence. Indent the comment the given number of
+    characters.
+
+    Doing this for sequences of maps results in nice comments that look like:
+
+    ```
+    things:
+        # First key description. Added by this function.
+        - key: foo
+          # Second key description. Added by add_comments_to_configuration_object().
+          other: bar
+    ```
     '''
-    for index, field_name in enumerate(config.keys()):
-        field_schema = schema['map'].get(field_name, {})
-        description = field_schema.get('desc')
+    if schema['items'].get('type') != 'object':
+        return
+
+    for field_name in config[0].keys():
+        field_schema = schema['items']['properties'].get(field_name, {})
+        description = field_schema.get('description')
 
         # No description to use? Skip it.
         if not field_schema or not description:
+            return
+
+        config[0].yaml_set_start_comment(description, indent=indent)
+
+        # We only want the first key's description here, as the rest of the keys get commented by
+        # add_comments_to_configuration_object().
+        return
+
+
+REQUIRED_SECTION_NAMES = {'location', 'retention'}
+REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
+COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
+
+
+def add_comments_to_configuration_object(config, schema, indent=0, skip_first=False):
+    '''
+    Using descriptions from a schema as a source, add those descriptions as comments to the given
+    config mapping, before each field. Indent the comment the given number of characters.
+    '''
+    for index, field_name in enumerate(config.keys()):
+        if skip_first and index == 0:
+            continue
+
+        field_schema = schema['properties'].get(field_name, {})
+        description = field_schema.get('description', '').strip()
+
+        # If this is an optional key, add an indicator to the comment flagging it to be commented
+        # out from the sample configuration. This sentinel is consumed by downstream processing that
+        # does the actual commenting out.
+        if field_name not in REQUIRED_SECTION_NAMES and field_name not in REQUIRED_KEYS:
+            description = (
+                '\n'.join((description, COMMENTED_OUT_SENTINEL))
+                if description
+                else COMMENTED_OUT_SENTINEL
+            )
+
+        # No description to use? Skip it.
+        if not field_schema or not description:  # pragma: no cover
             continue
 
         config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
+
         if index > 0:
             _insert_newline_before_comment(config, field_name)
 
 
-def generate_sample_configuration(config_filename, schema_filename):
+RUAMEL_YAML_COMMENTS_INDEX = 1
+
+
+def remove_commented_out_sentinel(config, field_name):
     '''
-    Given a target config filename and the path to a schema filename in pykwalify YAML schema
-    format, write out a sample configuration file based on that schema.
+    Given a configuration CommentedMap and a top-level field name in it, remove any "commented out"
+    sentinel found at the end of its YAML comments. This prevents the given field name from getting
+    commented out by downstream processing that consumes the sentinel.
+    '''
+    try:
+        last_comment_value = config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX][-1].value
+    except KeyError:
+        return
+
+    if last_comment_value == '# {}\n'.format(COMMENTED_OUT_SENTINEL):
+        config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX].pop()
+
+
+def merge_source_configuration_into_destination(destination_config, source_config):
+    '''
+    Deep merge the given source configuration dict into the destination configuration CommentedMap,
+    favoring values from the source when there are collisions.
+
+    The purpose of this is to upgrade configuration files from old versions of borgmatic by adding
+    new
+    configuration keys and comments.
+    '''
+    if not source_config:
+        return destination_config
+    if not destination_config or not isinstance(source_config, collections.abc.Mapping):
+        return source_config
+
+    for field_name, source_value in source_config.items():
+        # Since this key/value is from the source configuration, leave it uncommented and remove any
+        # sentinel that would cause it to get commented out.
+        remove_commented_out_sentinel(destination_config, field_name)
+
+        # This is a mapping. Recurse for this key/value.
+        if isinstance(source_value, collections.abc.Mapping):
+            destination_config[field_name] = merge_source_configuration_into_destination(
+                destination_config[field_name], source_value
+            )
+            continue
+
+        # This is a sequence. Recurse for each item in it.
+        if isinstance(source_value, collections.abc.Sequence) and not isinstance(source_value, str):
+            destination_value = destination_config[field_name]
+            destination_config[field_name] = yaml.comments.CommentedSeq(
+                [
+                    merge_source_configuration_into_destination(
+                        destination_value[index] if index < len(destination_value) else None,
+                        source_item,
+                    )
+                    for index, source_item in enumerate(source_value)
+                ]
+            )
+            continue
+
+        # This is some sort of scalar. Simply set it into the destination.
+        destination_config[field_name] = source_config[field_name]
+
+    return destination_config
+
+
+def generate_sample_configuration(source_filename, destination_filename, schema_filename):
+    '''
+    Given an optional source configuration filename, and a required destination configuration
+    filename, and the path to a schema filename in a YAML rendition of the JSON Schema format,
+    write out a sample configuration file based on that schema. If a source filename is provided,
+    merge the parsed contents of that configuration into the generated configuration.
     '''
     schema = yaml.round_trip_load(open(schema_filename))
-    config = _schema_to_sample_configuration(schema)
+    source_config = None
+
+    if source_filename:
+        source_config = load.load_configuration(source_filename)
+
+    destination_config = merge_source_configuration_into_destination(
+        _schema_to_sample_configuration(schema), source_config
+    )
 
     write_configuration(
-        config_filename, _comment_out_optional_configuration(_render_configuration(config))
+        destination_filename,
+        _comment_out_optional_configuration(render_configuration(destination_config)),
     )

borgmatic/config/normalize.py

@@ -0,0 +1,10 @@
+def normalize(config):
+    '''
+    Given a configuration dict, apply particular hard-coded rules to normalize its contents to
+    adhere to the configuration schema.
+    '''
+    exclude_if_present = config.get('location', {}).get('exclude_if_present')
+
+    # "Upgrade" exclude_if_present from a string to a list.
+    if isinstance(exclude_if_present, str):
+        config['location']['exclude_if_present'] = [exclude_if_present]

borgmatic/config/override.py

@@ -0,0 +1,71 @@
+import io
+
+import ruamel.yaml
+
+
+def set_values(config, keys, value):
+    '''
+    Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value,
+    descend into the hierarchy based on the keys to set the value into the right place.
+    '''
+    if not keys:
+        return
+
+    first_key = keys[0]
+    if len(keys) == 1:
+        config[first_key] = value
+        return
+
+    if first_key not in config:
+        config[first_key] = {}
+
+    set_values(config[first_key], keys[1:], value)
+
+
+def convert_value_type(value):
+    '''
+    Given a string value, determine its logical type (string, boolean, integer, etc.), and return it
+    converted to that type.
+    '''
+    return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
+
+
+def parse_overrides(raw_overrides):
+    '''
+    Given a sequence of configuration file override strings in the form of "section.option=value",
+    parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For
+    instance, given the following raw overrides:
+
+        ['section.my_option=value1', 'section.other_option=value2']
+
+    ... return this:
+
+        (
+            (('section', 'my_option'), 'value1'),
+            (('section', 'other_option'), 'value2'),
+        )
+
+    Raise ValueError if an override can't be parsed.
+    '''
+    if not raw_overrides:
+        return ()
+
+    try:
+        return tuple(
+            (tuple(raw_keys.split('.')), convert_value_type(value))
+            for raw_override in raw_overrides
+            for raw_keys, value in (raw_override.split('=', 1),)
+        )
+    except ValueError:
+        raise ValueError('Invalid override. Make sure you use the form: SECTION.OPTION=VALUE')
+
+
+def apply_overrides(config, raw_overrides):
+    '''
+    Given a sequence of configuration file override strings in the form of "section.option=value"
+    and a configuration dict, parse each override and set it the configuration dict.
+    '''
+    overrides = parse_overrides(raw_overrides)
+
+    for (keys, value) in overrides:
+        set_values(config, keys, value)

borgmatic/config/validate.py

@@ -1,11 +1,10 @@
-import logging
+import os
 
+import jsonschema
 import pkg_resources
-import pykwalify.core
-import pykwalify.errors
 import ruamel.yaml
 
-from borgmatic.config import load
+from borgmatic.config import load, normalize, override
 
 
 def schema_filename():
@@ -16,15 +15,40 @@ def schema_filename():
     return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
 
 
+def format_error_path_element(path_element):
+    '''
+    Given a path element into a JSON data structure, format it for display as a string.
+    '''
+    if isinstance(path_element, int):
+        return str('[{}]'.format(path_element))
+
+    return str('.{}'.format(path_element))
+
+
+def format_error(error):
+    '''
+    Given an instance of jsonschema.exceptions.ValidationError, format it for display as a string.
+    '''
+    if not error.path:
+        return 'At the top level: {}'.format(error.message)
+
+    formatted_path = ''.join(format_error_path_element(element) for element in error.path)
+    return "At '{}': {}".format(formatted_path.lstrip('.'), error.message)
+
+
 class Validation_error(ValueError):
     '''
-    A collection of error message strings generated when attempting to validate a particular
-    configurartion file.
+    A collection of error messages generated when attempting to validate a particular
+    configuration file.
     '''
 
-    def __init__(self, config_filename, error_messages):
+    def __init__(self, config_filename, errors):
+        '''
+        Given a configuration filename path and a sequence of
+        jsonschema.exceptions.ValidationError instances, create a Validation_error.
+        '''
         self.config_filename = config_filename
-        self.error_messages = error_messages
+        self.errors = errors
 
     def __str__(self):
         '''
@@ -32,7 +56,7 @@ class Validation_error(ValueError):
         '''
         return 'An error occurred while parsing a configuration file at {}:\n'.format(
             self.config_filename
-        ) + '\n'.join(self.error_messages)
+        ) + '\n'.join(format_error(error) for error in self.errors)
 
 
 def apply_logical_validation(config_filename, parsed_configuration):
@@ -64,11 +88,12 @@ def apply_logical_validation(config_filename, parsed_configuration):
             )
 
 
-def parse_configuration(config_filename, schema_filename):
+def parse_configuration(config_filename, schema_filename, overrides=None):
     '''
-    Given the path to a config filename in YAML format and the path to a schema filename in
-    pykwalify YAML schema format, return the parsed configuration as a data structure of nested
-    dicts and lists corresponding to the schema. Example return value:
+    Given the path to a config filename in YAML format, the path to a schema filename in a YAML
+    rendition of JSON Schema format, a sequence of configuration file override strings in the form
+    of "section.option=value", return the parsed configuration as a data structure of nested dicts
+    and lists corresponding to the schema. Example return value:
 
        {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
        'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
@@ -76,29 +101,42 @@ def parse_configuration(config_filename, schema_filename):
     Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
     have permissions to read the file, or Validation_error if the config does not match the schema.
     '''
-    logging.getLogger('pykwalify').setLevel(logging.ERROR)
-
     try:
         config = load.load_configuration(config_filename)
         schema = load.load_configuration(schema_filename)
     except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
-    # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
-    # remove all examples before passing the schema to pykwalify.
-    for section_name, section_schema in schema['map'].items():
-        for field_name, field_schema in section_schema['map'].items():
-            field_schema.pop('example', None)
+    override.apply_overrides(config, overrides)
+    normalize.normalize(config)
 
-    validator = pykwalify.core.Core(source_data=config, schema_data=schema)
-    parsed_result = validator.validate(raise_exception=False)
+    validator = jsonschema.Draft7Validator(schema)
+    validation_errors = tuple(validator.iter_errors(config))
 
-    if validator.validation_errors:
-        raise Validation_error(config_filename, validator.validation_errors)
+    if validation_errors:
+        raise Validation_error(config_filename, validation_errors)
 
-    apply_logical_validation(config_filename, parsed_result)
+    apply_logical_validation(config_filename, config)
 
-    return parsed_result
+    return config
+
+
+def normalize_repository_path(repository):
+    '''
+    Given a repository path, return the absolute path of it (for local repositories).
+    '''
+    # A colon in the repository indicates it's a remote repository. Bail.
+    if ':' in repository:
+        return repository
+
+    return os.path.abspath(repository)
+
+
+def repositories_match(first, second):
+    '''
+    Given two repository paths (relative and/or absolute), return whether they match.
+    '''
+    return normalize_repository_path(first) == normalize_repository_path(second)
 
 
 def guard_configuration_contains_repository(repository, configurations):
@@ -122,9 +160,7 @@ def guard_configuration_contains_repository(repository, configurations):
 
         if count > 1:
             raise ValueError(
-                'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
-                    repository
-                )
+                'Can\'t determine which repository to use. Use --repository option to disambiguate'
             )
 
         return
@@ -134,7 +170,7 @@ def guard_configuration_contains_repository(repository, configurations):
             config_repository
             for config in configurations.values()
             for config_repository in config['location']['repositories']
-            if repository == config_repository
+            if repositories_match(repository, config_repository)
         )
     )
 

borgmatic/execute.py

@@ -1,57 +1,281 @@
+import collections
 import logging
+import os
+import select
 import subprocess
 
 logger = logging.getLogger(__name__)
 
 
 ERROR_OUTPUT_MAX_LINE_COUNT = 25
+BORG_ERROR_EXIT_CODE = 2
 
 
-def execute_and_log_output(full_command, output_log_level, shell):
-    last_lines = []
-    process = subprocess.Popen(
-        full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell
-    )
+def exit_code_indicates_error(process, exit_code, borg_local_path=None):
+    '''
+    Return True if the given exit code from running a command corresponds to an error. If a Borg
+    local path is given and matches the process' command, then treat exit code 1 as a warning
+    instead of an error.
+    '''
+    if exit_code is None:
+        return False
 
-    while process.poll() is None:
-        line = process.stdout.readline().rstrip().decode()
-        if not line:
+    command = process.args.split(' ') if isinstance(process.args, str) else process.args
+
+    if borg_local_path and command[0] == borg_local_path:
+        return bool(exit_code < 0 or exit_code >= BORG_ERROR_EXIT_CODE)
+
+    return bool(exit_code != 0)
+
+
+def command_for_process(process):
+    '''
+    Given a process as an instance of subprocess.Popen, return the command string that was used to
+    invoke it.
+    '''
+    return process.args if isinstance(process.args, str) else ' '.join(process.args)
+
+
+def output_buffer_for_process(process, exclude_stdouts):
+    '''
+    Given a process as an instance of subprocess.Popen and a sequence of stdouts to exclude, return
+    either the process's stdout or stderr. The idea is that if stdout is excluded for a process, we
+    still have stderr to log.
+    '''
+    return process.stderr if process.stdout in exclude_stdouts else process.stdout
+
+
+def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
+    '''
+    Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
+    process with the requested log level. Additionally, raise a CalledProcessError if a process
+    exits with an error (or a warning for exit code 1, if that process matches the Borg local path).
+
+    For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
+    stdouts are given to exclude, then for any matching processes, log from their stderr instead.
+
+    Note that stdout for a process can be None if output is intentionally not captured. In which
+    case it won't be logged.
+    '''
+    # Map from output buffer to sequence of last lines.
+    buffer_last_lines = collections.defaultdict(list)
+    process_for_output_buffer = {
+        output_buffer_for_process(process, exclude_stdouts): process
+        for process in processes
+        if process.stdout or process.stderr
+    }
+    output_buffers = list(process_for_output_buffer.keys())
+
+    # Log output for each process until they all exit.
+    while True:
+        if output_buffers:
+            (ready_buffers, _, _) = select.select(output_buffers, [], [])
+
+            for ready_buffer in ready_buffers:
+                ready_process = process_for_output_buffer.get(ready_buffer)
+
+                # The "ready" process has exited, but it might be a pipe destination with other
+                # processes (pipe sources) waiting to be read from. So as a measure to prevent
+                # hangs, vent all processes when one exits.
+                if ready_process and ready_process.poll() is not None:
+                    for other_process in processes:
+                        if (
+                            other_process.poll() is None
+                            and other_process.stdout
+                            and other_process.stdout not in output_buffers
+                        ):
+                            # Add the process's output to output_buffers to ensure it'll get read.
+                            output_buffers.append(other_process.stdout)
+
+                line = ready_buffer.readline().rstrip().decode()
+                if not line or not ready_process:
+                    continue
+
+                # Keep the last few lines of output in case the process errors, and we need the output for
+                # the exception below.
+                last_lines = buffer_last_lines[ready_buffer]
+                last_lines.append(line)
+                if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+                    last_lines.pop(0)
+
+                logger.log(output_log_level, line)
+
+        still_running = False
+
+        for process in processes:
+            exit_code = process.poll() if output_buffers else process.wait()
+
+            if exit_code is None:
+                still_running = True
+
+            # If any process errors, then raise accordingly.
+            if exit_code_indicates_error(process, exit_code, borg_local_path):
+                # If an error occurs, include its output in the raised exception so that we don't
+                # inadvertently hide error output.
+                output_buffer = output_buffer_for_process(process, exclude_stdouts)
+
+                last_lines = buffer_last_lines[output_buffer] if output_buffer else []
+                if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
+                    last_lines.insert(0, '...')
+
+                # Something has gone wrong. So vent each process' output buffer to prevent it from
+                # hanging. And then kill the process.
+                for other_process in processes:
+                    if other_process.poll() is None:
+                        other_process.stdout.read(0)
+                        other_process.kill()
+
+                raise subprocess.CalledProcessError(
+                    exit_code, command_for_process(process), '\n'.join(last_lines)
+                )
+
+        if not still_running:
+            break
+
+    # Consume any remaining output that we missed (if any).
+    for process in processes:
+        output_buffer = output_buffer_for_process(process, exclude_stdouts)
+
+        if not output_buffer:
             continue
 
-        # Keep the last few lines of output in case the command errors, and we need the output for
-        # the exception below.
-        last_lines.append(line)
-        if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
-            last_lines.pop(0)
+        while True:  # pragma: no cover
+            remaining_output = output_buffer.readline().rstrip().decode()
 
-        logger.log(output_log_level, line)
+            if not remaining_output:
+                break
 
-    remaining_output = process.stdout.read().rstrip().decode()
-    if remaining_output:  # pragma: no cover
-        logger.log(output_log_level, remaining_output)
-
-    exit_code = process.poll()
-    if exit_code != 0:
-        # If an error occurs, include its output in the raised exception so that we don't
-        # inadvertently hide error output.
-        if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
-            last_lines.insert(0, '...')
-
-        raise subprocess.CalledProcessError(
-            exit_code, ' '.join(full_command), '\n'.join(last_lines)
-        )
+            logger.log(output_log_level, remaining_output)
 
 
-def execute_command(full_command, output_log_level=logging.INFO, shell=False):
+def log_command(full_command, input_file, output_file):
+    '''
+    Log the given command (a sequence of command/argument strings), along with its input/output file
+    paths.
+    '''
+    logger.debug(
+        ' '.join(full_command)
+        + (' < {}'.format(getattr(input_file, 'name', '')) if input_file else '')
+        + (' > {}'.format(getattr(output_file, 'name', '')) if output_file else '')
+    )
+
+
+# An sentinel passed as an output file to execute_command() to indicate that the command's output
+# should be allowed to flow through to stdout without being captured for logging. Useful for
+# commands with interactive prompts or those that mess directly with the console.
+DO_NOT_CAPTURE = object()
+
+
+def execute_command(
+    full_command,
+    output_log_level=logging.INFO,
+    output_file=None,
+    input_file=None,
+    shell=False,
+    extra_environment=None,
+    working_directory=None,
+    borg_local_path=None,
+    run_to_completion=True,
+):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
-    given log level. If output log level is None, instead capture and return the output. If
-    shell is True, execute the command within a shell.
+    given log level. If output log level is None, instead capture and return the output. (Implies
+    run_to_completion.) If an open output file object is given, then write stdout to the file and
+    only log stderr (but only if an output log level is set). If an open input file object is given,
+    then read stdin from the file. If shell is True, execute the command within a shell. If an extra
+    environment dict is given, then use it to augment the current environment, and pass the result
+    into the command. If a working directory is given, use that as the present working directory
+    when running the command. If a Borg local path is given, and the command matches it (regardless
+    of arguments), treat exit code 1 as a warning instead of an error. If run to completion is
+    False, then return the process for the command without executing it to completion.
+
+    Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
-    logger.debug(' '.join(full_command))
+    log_command(full_command, input_file, output_file)
+    environment = {**os.environ, **extra_environment} if extra_environment else None
+    do_not_capture = bool(output_file is DO_NOT_CAPTURE)
+    command = ' '.join(full_command) if shell else full_command
 
     if output_log_level is None:
-        output = subprocess.check_output(full_command, shell=shell)
+        output = subprocess.check_output(
+            command, shell=shell, env=environment, cwd=working_directory
+        )
         return output.decode() if output is not None else None
-    else:
-        execute_and_log_output(full_command, output_log_level, shell=shell)
+
+    process = subprocess.Popen(
+        command,
+        stdin=input_file,
+        stdout=None if do_not_capture else (output_file or subprocess.PIPE),
+        stderr=None if do_not_capture else (subprocess.PIPE if output_file else subprocess.STDOUT),
+        shell=shell,
+        env=environment,
+        cwd=working_directory,
+    )
+    if not run_to_completion:
+        return process
+
+    log_outputs(
+        (process,), (input_file, output_file), output_log_level, borg_local_path=borg_local_path
+    )
+
+
+def execute_command_with_processes(
+    full_command,
+    processes,
+    output_log_level=logging.INFO,
+    output_file=None,
+    input_file=None,
+    shell=False,
+    extra_environment=None,
+    working_directory=None,
+    borg_local_path=None,
+):
+    '''
+    Execute the given command (a sequence of command/argument strings) and log its output at the
+    given log level. Simultaneously, continue to poll one or more active processes so that they
+    run as well. This is useful, for instance, for processes that are streaming output to a named
+    pipe that the given command is consuming from.
+
+    If an open output file object is given, then write stdout to the file and only log stderr (but
+    only if an output log level is set). If an open input file object is given, then read stdin from
+    the file.  If shell is True, execute the command within a shell. If an extra environment dict is
+    given, then use it to augment the current environment, and pass the result into the command. If
+    a working directory is given, use that as the present working directory when running the
+    command. If a Borg local path is given, then for any matching command or process (regardless of
+    arguments), treat exit code 1 as a warning instead of an error.
+
+    Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
+    upstream process.
+    '''
+    log_command(full_command, input_file, output_file)
+    environment = {**os.environ, **extra_environment} if extra_environment else None
+    do_not_capture = bool(output_file is DO_NOT_CAPTURE)
+    command = ' '.join(full_command) if shell else full_command
+
+    try:
+        command_process = subprocess.Popen(
+            command,
+            stdin=input_file,
+            stdout=None if do_not_capture else (output_file or subprocess.PIPE),
+            stderr=None
+            if do_not_capture
+            else (subprocess.PIPE if output_file else subprocess.STDOUT),
+            shell=shell,
+            env=environment,
+            cwd=working_directory,
+        )
+    except (subprocess.CalledProcessError, OSError):
+        # Something has gone wrong. So vent each process' output buffer to prevent it from hanging.
+        # And then kill the process.
+        for process in processes:
+            if process.poll() is None:
+                process.stdout.read(0)
+                process.kill()
+        raise
+
+    log_outputs(
+        tuple(processes) + (command_process,),
+        (input_file, output_file),
+        output_log_level,
+        borg_local_path=borg_local_path,
+    )

borgmatic/hooks/command.py

@@ -6,13 +6,31 @@ from borgmatic import execute
 logger = logging.getLogger(__name__)
 
 
-def execute_hook(commands, umask, config_filename, description, dry_run):
+SOFT_FAIL_EXIT_CODE = 75
+
+
+def interpolate_context(command, context):
+    '''
+    Given a single hook command and a dict of context names/values, interpolate the values by
+    "{name}" into the command and return the result.
+    '''
+    for name, value in context.items():
+        command = command.replace('{%s}' % name, str(value))
+
+    return command
+
+
+def execute_hook(commands, umask, config_filename, description, dry_run, **context):
     '''
     Given a list of hook commands to execute, a umask to execute with (or None), a config filename,
     a hook description, and whether this is a dry run, run the given commands. Or, don't run them
     if this is a dry run.
 
+    The context contains optional values interpolated by name into the hook commands. Currently,
+    this only applies to the on_error hook.
+
     Raise ValueError if the umask cannot be parsed.
+    Raise subprocesses.CalledProcessError if an error occurs in a hook.
     '''
     if not commands:
         logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
@@ -20,6 +38,9 @@ def execute_hook(commands, umask, config_filename, description, dry_run):
 
     dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
 
+    context['configuration_filename'] = config_filename
+    commands = [interpolate_context(command, context) for command in commands]
+
     if len(commands) == 1:
         logger.info(
             '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label)
@@ -51,3 +72,24 @@ def execute_hook(commands, umask, config_filename, description, dry_run):
     finally:
         if original_umask:
             os.umask(original_umask)
+
+
+def considered_soft_failure(config_filename, error):
+    '''
+    Given a configuration filename and an exception object, return whether the exception object
+    represents a subprocess.CalledProcessError with a return code of SOFT_FAIL_EXIT_CODE. If so,
+    that indicates that the error is a "soft failure", and should not result in an error.
+    '''
+    exit_code = getattr(error, 'returncode', None)
+    if exit_code is None:
+        return False
+
+    if exit_code == SOFT_FAIL_EXIT_CODE:
+        logger.info(
+            '{}: Command hook exited with soft failure exit code ({}); skipping remaining actions'.format(
+                config_filename, SOFT_FAIL_EXIT_CODE
+            )
+        )
+        return True
+
+    return False

borgmatic/hooks/cronhub.py

@@ -0,0 +1,50 @@
+import logging
+
+import requests
+
+from borgmatic.hooks import monitor
+
+logger = logging.getLogger(__name__)
+
+MONITOR_STATE_TO_CRONHUB = {
+    monitor.State.START: 'start',
+    monitor.State.FINISH: 'finish',
+    monitor.State.FAIL: 'fail',
+}
+
+
+def initialize_monitor(
+    ping_url, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
+def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
+    '''
+    Ping the given Cronhub URL, modified with the monitor.State. Use the given configuration
+    filename in any log entries. If this is a dry run, then don't actually ping anything.
+    '''
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+    formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state])
+    ping_url = ping_url.replace('/start/', formatted_state).replace('/ping/', formatted_state)
+
+    logger.info(
+        '{}: Pinging Cronhub {}{}'.format(config_filename, state.name.lower(), dry_run_label)
+    )
+    logger.debug('{}: Using Cronhub ping URL {}'.format(config_filename, ping_url))
+
+    if not dry_run:
+        logging.getLogger('urllib3').setLevel(logging.ERROR)
+        requests.get(ping_url)
+
+
+def destroy_monitor(
+    ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No destruction is necessary for this monitor.
+    '''
+    pass

borgmatic/hooks/cronitor.py

@@ -0,0 +1,49 @@
+import logging
+
+import requests
+
+from borgmatic.hooks import monitor
+
+logger = logging.getLogger(__name__)
+
+MONITOR_STATE_TO_CRONITOR = {
+    monitor.State.START: 'run',
+    monitor.State.FINISH: 'complete',
+    monitor.State.FAIL: 'fail',
+}
+
+
+def initialize_monitor(
+    ping_url, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
+def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
+    '''
+    Ping the given Cronitor URL, modified with the monitor.State. Use the given configuration
+    filename in any log entries. If this is a dry run, then don't actually ping anything.
+    '''
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+    ping_url = '{}/{}'.format(ping_url, MONITOR_STATE_TO_CRONITOR[state])
+
+    logger.info(
+        '{}: Pinging Cronitor {}{}'.format(config_filename, state.name.lower(), dry_run_label)
+    )
+    logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
+
+    if not dry_run:
+        logging.getLogger('urllib3').setLevel(logging.ERROR)
+        requests.get(ping_url)
+
+
+def destroy_monitor(
+    ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No destruction is necessary for this monitor.
+    '''
+    pass

borgmatic/hooks/dispatch.py

@@ -0,0 +1,62 @@
+import logging
+
+from borgmatic.hooks import cronhub, cronitor, healthchecks, mysql, pagerduty, postgresql
+
+logger = logging.getLogger(__name__)
+
+HOOK_NAME_TO_MODULE = {
+    'healthchecks': healthchecks,
+    'cronitor': cronitor,
+    'cronhub': cronhub,
+    'pagerduty': pagerduty,
+    'postgresql_databases': postgresql,
+    'mysql_databases': mysql,
+}
+
+
+def call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs):
+    '''
+    Given the hooks configuration dict and a prefix to use in log entries, call the requested
+    function of the Python module corresponding to the given hook name. Supply that call with the
+    configuration for this hook, the log prefix, and any given args and kwargs. Return any return
+    value.
+
+    If the hook name is not present in the hooks configuration, then bail without calling anything.
+
+    Raise ValueError if the hook name is unknown.
+    Raise AttributeError if the function name is not found in the module.
+    Raise anything else that the called function raises.
+    '''
+    config = hooks.get(hook_name)
+    if not config:
+        logger.debug('{}: No {} hook configured.'.format(log_prefix, hook_name))
+        return
+
+    try:
+        module = HOOK_NAME_TO_MODULE[hook_name]
+    except KeyError:
+        raise ValueError('Unknown hook name: {}'.format(hook_name))
+
+    logger.debug('{}: Calling {} hook function {}'.format(log_prefix, hook_name, function_name))
+    return getattr(module, function_name)(config, log_prefix, *args, **kwargs)
+
+
+def call_hooks(function_name, hooks, log_prefix, hook_names, *args, **kwargs):
+    '''
+    Given the hooks configuration dict and a prefix to use in log entries, call the requested
+    function of the Python module corresponding to each given hook name. Supply each call with the
+    configuration for that hook, the log prefix, and any given args and kwargs. Collect any return
+    values into a dict from hook name to return value.
+
+    If the hook name is not present in the hooks configuration, then don't call the function for it,
+    and omit it from the return values.
+
+    Raise ValueError if the hook name is unknown.
+    Raise AttributeError if the function name is not found in the module.
+    Raise anything else that a called function raises. An error stops calls to subsequent functions.
+    '''
+    return {
+        hook_name: call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs)
+        for hook_name in hook_names
+        if hooks.get(hook_name)
+    }

borgmatic/hooks/dump.py

@@ -0,0 +1,76 @@
+import logging
+import os
+import shutil
+
+from borgmatic.borg.create import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+
+logger = logging.getLogger(__name__)
+
+DATABASE_HOOK_NAMES = ('postgresql_databases', 'mysql_databases')
+
+
+def make_database_dump_path(borgmatic_source_directory, database_hook_name):
+    '''
+    Given a borgmatic source directory (or None) and a database hook name, construct a database dump
+    path.
+    '''
+    if not borgmatic_source_directory:
+        borgmatic_source_directory = DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+
+    return os.path.join(borgmatic_source_directory, database_hook_name)
+
+
+def make_database_dump_filename(dump_path, name, hostname=None):
+    '''
+    Based on the given dump directory path, database name, and hostname, return a filename to use
+    for the database dump. The hostname defaults to localhost.
+
+    Raise ValueError if the database name is invalid.
+    '''
+    if os.path.sep in name:
+        raise ValueError('Invalid database name {}'.format(name))
+
+    return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
+
+
+def create_parent_directory_for_dump(dump_path):
+    '''
+    Create a directory to contain the given dump path.
+    '''
+    os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
+
+
+def create_named_pipe_for_dump(dump_path):
+    '''
+    Create a named pipe at the given dump path.
+    '''
+    create_parent_directory_for_dump(dump_path)
+    os.mkfifo(dump_path, mode=0o600)
+
+
+def remove_database_dumps(dump_path, database_type_name, log_prefix, dry_run):
+    '''
+    Remove all database dumps in the given dump directory path (including the directory itself). If
+    this is a dry run, then don't actually remove anything.
+    '''
+    dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
+
+    logger.info(
+        '{}: Removing {} database dumps{}'.format(log_prefix, database_type_name, dry_run_label)
+    )
+
+    expanded_path = os.path.expanduser(dump_path)
+
+    if dry_run:
+        return
+
+    if os.path.exists(expanded_path):
+        shutil.rmtree(expanded_path)
+
+
+def convert_glob_patterns_to_borg_patterns(patterns):
+    '''
+    Convert a sequence of shell glob patterns like "/etc/*" to the corresponding Borg archive
+    patterns like "sh:etc/*".
+    '''
+    return ['sh:{}'.format(pattern.lstrip(os.path.sep)) for pattern in patterns]

borgmatic/hooks/healthchecks.py

@@ -0,0 +1,121 @@
+import logging
+
+import requests
+
+from borgmatic.hooks import monitor
+
+logger = logging.getLogger(__name__)
+
+MONITOR_STATE_TO_HEALTHCHECKS = {
+    monitor.State.START: 'start',
+    monitor.State.FINISH: None,  # Healthchecks doesn't append to the URL for the finished state.
+    monitor.State.FAIL: 'fail',
+}
+
+PAYLOAD_TRUNCATION_INDICATOR = '...\n'
+PAYLOAD_LIMIT_BYTES = 10 * 1024 - len(PAYLOAD_TRUNCATION_INDICATOR)
+
+
+class Forgetful_buffering_handler(logging.Handler):
+    '''
+    A buffering log handler that stores log messages in memory, and throws away messages (oldest
+    first) once a particular capacity in bytes is reached.
+    '''
+
+    def __init__(self, byte_capacity, log_level):
+        super().__init__()
+
+        self.byte_capacity = byte_capacity
+        self.byte_count = 0
+        self.buffer = []
+        self.forgot = False
+        self.setLevel(log_level)
+
+    def emit(self, record):
+        message = record.getMessage() + '\n'
+        self.byte_count += len(message)
+        self.buffer.append(message)
+
+        while self.byte_count > self.byte_capacity and self.buffer:
+            self.byte_count -= len(self.buffer[0])
+            self.buffer.pop(0)
+            self.forgot = True
+
+
+def format_buffered_logs_for_payload():
+    '''
+    Get the handler previously added to the root logger, and slurp buffered logs out of it to
+    send to Healthchecks.
+    '''
+    try:
+        buffering_handler = next(
+            handler
+            for handler in logging.getLogger().handlers
+            if isinstance(handler, Forgetful_buffering_handler)
+        )
+    except StopIteration:
+        # No handler means no payload.
+        return ''
+
+    payload = ''.join(message for message in buffering_handler.buffer)
+
+    if buffering_handler.forgot:
+        return PAYLOAD_TRUNCATION_INDICATOR + payload
+
+    return payload
+
+
+def initialize_monitor(
+    ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    Add a handler to the root logger that stores in memory the most recent logs emitted. That
+    way, we can send them all to Healthchecks upon a finish or failure state.
+    '''
+    logging.getLogger().addHandler(
+        Forgetful_buffering_handler(PAYLOAD_LIMIT_BYTES, monitoring_log_level)
+    )
+
+
+def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level, dry_run):
+    '''
+    Ping the given Healthchecks URL or UUID, modified with the monitor.State. Use the given
+    configuration filename in any log entries, and log to Healthchecks with the giving log level.
+    If this is a dry run, then don't actually ping anything.
+    '''
+    ping_url = (
+        ping_url_or_uuid
+        if ping_url_or_uuid.startswith('http')
+        else 'https://hc-ping.com/{}'.format(ping_url_or_uuid)
+    )
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+
+    healthchecks_state = MONITOR_STATE_TO_HEALTHCHECKS.get(state)
+    if healthchecks_state:
+        ping_url = '{}/{}'.format(ping_url, healthchecks_state)
+
+    logger.info(
+        '{}: Pinging Healthchecks {}{}'.format(config_filename, state.name.lower(), dry_run_label)
+    )
+    logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
+
+    if state in (monitor.State.FINISH, monitor.State.FAIL):
+        payload = format_buffered_logs_for_payload()
+    else:
+        payload = ''
+
+    if not dry_run:
+        logging.getLogger('urllib3').setLevel(logging.ERROR)
+        requests.post(ping_url, data=payload.encode('utf-8'))
+
+
+def destroy_monitor(ping_url_or_uuid, config_filename, monitoring_log_level, dry_run):
+    '''
+    Remove the monitor handler that was added to the root logger. This prevents the handler from
+    getting reused by other instances of this monitor.
+    '''
+    logger = logging.getLogger()
+
+    for handler in tuple(logger.handlers):
+        if isinstance(handler, Forgetful_buffering_handler):
+            logger.removeHandler(handler)

borgmatic/hooks/monitor.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty')
+
+
+class State(Enum):
+    START = 1
+    FINISH = 2
+    FAIL = 3

borgmatic/hooks/mysql.py

@@ -0,0 +1,175 @@
+import logging
+
+from borgmatic.execute import execute_command, execute_command_with_processes
+from borgmatic.hooks import dump
+
+logger = logging.getLogger(__name__)
+
+
+def make_dump_path(location_config):  # pragma: no cover
+    '''
+    Make the dump path from the given location configuration and the name of this hook.
+    '''
+    return dump.make_database_dump_path(
+        location_config.get('borgmatic_source_directory'), 'mysql_databases'
+    )
+
+
+SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
+
+
+def database_names_to_dump(database, extra_environment, log_prefix, dry_run_label):
+    '''
+    Given a requested database name, return the corresponding sequence of database names to dump.
+    In the case of "all", query for the names of databases on the configured host and return them,
+    excluding any system databases that will cause problems during restore.
+    '''
+    requested_name = database['name']
+
+    if requested_name != 'all':
+        return (requested_name,)
+
+    show_command = (
+        ('mysql',)
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--user', database['username']) if 'username' in database else ())
+        + ('--skip-column-names', '--batch')
+        + ('--execute', 'show schemas')
+    )
+    logger.debug(
+        '{}: Querying for "all" MySQL databases to dump{}'.format(log_prefix, dry_run_label)
+    )
+    show_output = execute_command(
+        show_command, output_log_level=None, extra_environment=extra_environment
+    )
+
+    return tuple(
+        show_name
+        for show_name in show_output.strip().splitlines()
+        if show_name not in SYSTEM_DATABASE_NAMES
+    )
+
+
+def dump_databases(databases, log_prefix, location_config, dry_run):
+    '''
+    Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence
+    of dicts, one dict describing each database as per the configuration schema. Use the given log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path.
+
+    Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
+    pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
+    '''
+    dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+    processes = []
+
+    logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
+
+    for database in databases:
+        requested_name = database['name']
+        dump_filename = dump.make_database_dump_filename(
+            make_dump_path(location_config), requested_name, database.get('hostname')
+        )
+        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+        dump_database_names = database_names_to_dump(
+            database, extra_environment, log_prefix, dry_run_label
+        )
+        if not dump_database_names:
+            raise ValueError('Cannot find any MySQL databases to dump.')
+
+        dump_command = (
+            ('mysqldump',)
+            + ('--add-drop-database',)
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+            + (('--user', database['username']) if 'username' in database else ())
+            + (tuple(database['options'].split(' ')) if 'options' in database else ())
+            + ('--databases',)
+            + dump_database_names
+            # Use shell redirection rather than execute_command(output_file=open(...)) to prevent
+            # the open() call on a named pipe from hanging the main borgmatic process.
+            + ('>', dump_filename)
+        )
+
+        logger.debug(
+            '{}: Dumping MySQL database {} to {}{}'.format(
+                log_prefix, requested_name, dump_filename, dry_run_label
+            )
+        )
+        if dry_run:
+            continue
+
+        dump.create_named_pipe_for_dump(dump_filename)
+
+        processes.append(
+            execute_command(
+                dump_command,
+                shell=True,
+                extra_environment=extra_environment,
+                run_to_completion=False,
+            )
+        )
+
+    return processes
+
+
+def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
+    '''
+    Remove all database dump files for this hook regardless of the given databases. Use the log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path. If this is a dry run, then don't actually remove anything.
+    '''
+    dump.remove_database_dumps(make_dump_path(location_config), 'MySQL', log_prefix, dry_run)
+
+
+def make_database_dump_pattern(
+    databases, log_prefix, location_config, name=None
+):  # pragma: no cover
+    '''
+    Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
+    and a database name to match, return the corresponding glob patterns to match the database dump
+    in an archive.
+    '''
+    return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
+
+
+def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+    '''
+    Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
+    one-element sequence containing a dict describing the database, as per the configuration schema.
+    Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
+    anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
+    output to consume.
+    '''
+    dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
+
+    if len(database_config) != 1:
+        raise ValueError('The database configuration value is invalid')
+
+    database = database_config[0]
+    restore_command = (
+        ('mysql', '--batch', '--verbose')
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--user', database['username']) if 'username' in database else ())
+    )
+    extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+
+    logger.debug(
+        '{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
+    )
+    if dry_run:
+        return
+
+    execute_command_with_processes(
+        restore_command,
+        [extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=extra_environment,
+        borg_local_path=location_config.get('local_path', 'borg'),
+    )

borgmatic/hooks/pagerduty.py

@@ -0,0 +1,80 @@
+import datetime
+import json
+import logging
+import platform
+
+import requests
+
+from borgmatic.hooks import monitor
+
+logger = logging.getLogger(__name__)
+
+EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue'
+
+
+def initialize_monitor(
+    integration_key, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
+def ping_monitor(integration_key, config_filename, state, monitoring_log_level, dry_run):
+    '''
+    If this is an error state, create a PagerDuty event with the given integration key. Use the
+    given configuration filename in any log entries. If this is a dry run, then don't actually
+    create an event.
+    '''
+    if state != monitor.State.FAIL:
+        logger.debug(
+            '{}: Ignoring unsupported monitoring {} in PagerDuty hook'.format(
+                config_filename, state.name.lower()
+            )
+        )
+        return
+
+    dry_run_label = ' (dry run; not actually sending)' if dry_run else ''
+    logger.info('{}: Sending failure event to PagerDuty {}'.format(config_filename, dry_run_label))
+
+    if dry_run:
+        return
+
+    hostname = platform.node()
+    local_timestamp = (
+        datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).astimezone().isoformat()
+    )
+    payload = json.dumps(
+        {
+            'routing_key': integration_key,
+            'event_action': 'trigger',
+            'payload': {
+                'summary': 'backup failed on {}'.format(hostname),
+                'severity': 'error',
+                'source': hostname,
+                'timestamp': local_timestamp,
+                'component': 'borgmatic',
+                'group': 'backups',
+                'class': 'backup failure',
+                'custom_details': {
+                    'hostname': hostname,
+                    'configuration filename': config_filename,
+                    'server time': local_timestamp,
+                },
+            },
+        }
+    )
+    logger.debug('{}: Using PagerDuty payload: {}'.format(config_filename, payload))
+
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    requests.post(EVENTS_API_URL, data=payload.encode('utf-8'))
+
+
+def destroy_monitor(
+    ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No destruction is necessary for this monitor.
+    '''
+    pass

borgmatic/hooks/postgresql.py

@@ -0,0 +1,179 @@
+import logging
+
+from borgmatic.execute import execute_command, execute_command_with_processes
+from borgmatic.hooks import dump
+
+logger = logging.getLogger(__name__)
+
+
+def make_dump_path(location_config):  # pragma: no cover
+    '''
+    Make the dump path from the given location configuration and the name of this hook.
+    '''
+    return dump.make_database_dump_path(
+        location_config.get('borgmatic_source_directory'), 'postgresql_databases'
+    )
+
+
+def make_extra_environment(database):
+    '''
+    Make the extra_environment dict from the given database configuration.
+    '''
+    extra = dict()
+    if 'password' in database:
+        extra['PGPASSWORD'] = database['password']
+    extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
+    if 'ssl_cert' in database:
+        extra['PGSSLCERT'] = database['ssl_cert']
+    if 'ssl_key' in database:
+        extra['PGSSLKEY'] = database['ssl_key']
+    if 'ssl_root_cert' in database:
+        extra['PGSSLROOTCERT'] = database['ssl_root_cert']
+    if 'ssl_crl' in database:
+        extra['PGSSLCRL'] = database['ssl_crl']
+    return extra
+
+
+def dump_databases(databases, log_prefix, location_config, dry_run):
+    '''
+    Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the given log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path.
+
+    Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
+    pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
+    '''
+    dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+    processes = []
+
+    logger.info('{}: Dumping PostgreSQL databases{}'.format(log_prefix, dry_run_label))
+
+    for database in databases:
+        name = database['name']
+        dump_filename = dump.make_database_dump_filename(
+            make_dump_path(location_config), name, database.get('hostname')
+        )
+        all_databases = bool(name == 'all')
+        dump_format = database.get('format', 'custom')
+        command = (
+            (
+                'pg_dumpall' if all_databases else 'pg_dump',
+                '--no-password',
+                '--clean',
+                '--if-exists',
+            )
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--username', database['username']) if 'username' in database else ())
+            + (() if all_databases else ('--format', dump_format))
+            + (('--file', dump_filename) if dump_format == 'directory' else ())
+            + (tuple(database['options'].split(' ')) if 'options' in database else ())
+            + (() if all_databases else (name,))
+            # Use shell redirection rather than the --file flag to sidestep synchronization issues
+            # when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump
+            # format in a particular, a named destination is required, and redirection doesn't work.
+            + (('>', dump_filename) if dump_format != 'directory' else ())
+        )
+        extra_environment = make_extra_environment(database)
+
+        logger.debug(
+            '{}: Dumping PostgreSQL database {} to {}{}'.format(
+                log_prefix, name, dump_filename, dry_run_label
+            )
+        )
+        if dry_run:
+            continue
+
+        if dump_format == 'directory':
+            dump.create_parent_directory_for_dump(dump_filename)
+        else:
+            dump.create_named_pipe_for_dump(dump_filename)
+
+        processes.append(
+            execute_command(
+                command, shell=True, extra_environment=extra_environment, run_to_completion=False
+            )
+        )
+
+    return processes
+
+
+def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
+    '''
+    Remove all database dump files for this hook regardless of the given databases. Use the log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path. If this is a dry run, then don't actually remove anything.
+    '''
+    dump.remove_database_dumps(make_dump_path(location_config), 'PostgreSQL', log_prefix, dry_run)
+
+
+def make_database_dump_pattern(
+    databases, log_prefix, location_config, name=None
+):  # pragma: no cover
+    '''
+    Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
+    and a database name to match, return the corresponding glob patterns to match the database dump
+    in an archive.
+    '''
+    return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
+
+
+def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+    '''
+    Restore the given PostgreSQL database from an extract stream. The database is supplied as a
+    one-element sequence containing a dict describing the database, as per the configuration schema.
+    Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
+    anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
+    output to consume.
+
+    If the extract process is None, then restore the dump from the filesystem rather than from an
+    extract stream.
+    '''
+    dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
+
+    if len(database_config) != 1:
+        raise ValueError('The database configuration value is invalid')
+
+    database = database_config[0]
+    all_databases = bool(database['name'] == 'all')
+    dump_filename = dump.make_database_dump_filename(
+        make_dump_path(location_config), database['name'], database.get('hostname')
+    )
+    analyze_command = (
+        ('psql', '--no-password', '--quiet')
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--username', database['username']) if 'username' in database else ())
+        + (('--dbname', database['name']) if not all_databases else ())
+        + ('--command', 'ANALYZE')
+    )
+    restore_command = (
+        ('psql' if all_databases else 'pg_restore', '--no-password')
+        + (
+            ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
+            if not all_databases
+            else ()
+        )
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--username', database['username']) if 'username' in database else ())
+        + (() if extract_process else (dump_filename,))
+    )
+    extra_environment = make_extra_environment(database)
+
+    logger.debug(
+        '{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
+    )
+    if dry_run:
+        return
+
+    execute_command_with_processes(
+        restore_command,
+        [extract_process] if extract_process else [],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout if extract_process else None,
+        extra_environment=extra_environment,
+        borg_local_path=location_config.get('local_path', 'borg'),
+    )
+    execute_command(analyze_command, extra_environment=extra_environment)

borgmatic/logger.py

@@ -1,4 +1,5 @@
 import logging
+import logging.handlers
 import os
 import sys
 
@@ -26,7 +27,7 @@ def interactive_console():
     Return whether the current console is "interactive". Meaning: Capable of
     user input and not just something like a cron job.
     '''
-    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+    return sys.stderr.isatty() and os.environ.get('TERM') != 'dumb'
 
 
 def should_do_markup(no_color, configs):
@@ -48,6 +49,42 @@ def should_do_markup(no_color, configs):
     return interactive_console()
 
 
+class Multi_stream_handler(logging.Handler):
+    '''
+    A logging handler that dispatches each log record to one of multiple stream handlers depending
+    on the record's log level.
+    '''
+
+    def __init__(self, log_level_to_stream_handler):
+        super(Multi_stream_handler, self).__init__()
+        self.log_level_to_handler = log_level_to_stream_handler
+        self.handlers = set(self.log_level_to_handler.values())
+
+    def flush(self):  # pragma: no cover
+        super(Multi_stream_handler, self).flush()
+
+        for handler in self.handlers:
+            handler.flush()
+
+    def emit(self, record):
+        '''
+        Dispatch the log record to the approriate stream handler for the record's log level.
+        '''
+        self.log_level_to_handler[record.levelno].emit(record)
+
+    def setFormatter(self, formatter):  # pragma: no cover
+        super(Multi_stream_handler, self).setFormatter(formatter)
+
+        for handler in self.handlers:
+            handler.setFormatter(formatter)
+
+    def setLevel(self, level):  # pragma: no cover
+        super(Multi_stream_handler, self).setLevel(level)
+
+        for handler in self.handlers:
+            handler.setLevel(level)
+
+
 LOG_LEVEL_TO_COLOR = {
     logging.CRITICAL: colorama.Fore.RED,
     logging.ERROR: colorama.Fore.RED,
@@ -73,29 +110,65 @@ def color_text(color, message):
     return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL)
 
 
-def configure_logging(console_log_level, syslog_log_level=None):
+def configure_logging(
+    console_log_level,
+    syslog_log_level=None,
+    log_file_log_level=None,
+    monitoring_log_level=None,
+    log_file=None,
+):
     '''
-    Configure logging to go to both the console and syslog. Use the given log levels, respectively.
+    Configure logging to go to both the console and (syslog or log file). Use the given log levels,
+    respectively.
+
+    Raise FileNotFoundError or PermissionError if the log file could not be opened for writing.
     '''
     if syslog_log_level is None:
         syslog_log_level = console_log_level
+    if log_file_log_level is None:
+        log_file_log_level = console_log_level
+    if monitoring_log_level is None:
+        monitoring_log_level = console_log_level
 
-    console_handler = logging.StreamHandler()
+    # Log certain log levels to console stderr and others to stdout. This supports use cases like
+    # grepping (non-error) output.
+    console_error_handler = logging.StreamHandler(sys.stderr)
+    console_standard_handler = logging.StreamHandler(sys.stdout)
+    console_handler = Multi_stream_handler(
+        {
+            logging.CRITICAL: console_error_handler,
+            logging.ERROR: console_error_handler,
+            logging.WARN: console_standard_handler,
+            logging.INFO: console_standard_handler,
+            logging.DEBUG: console_standard_handler,
+        }
+    )
     console_handler.setFormatter(Console_color_formatter())
     console_handler.setLevel(console_log_level)
 
     syslog_path = None
-    if os.path.exists('/dev/log'):
-        syslog_path = '/dev/log'
-    elif os.path.exists('/var/run/syslog'):
-        syslog_path = '/var/run/syslog'
+    if log_file is None:
+        if os.path.exists('/dev/log'):
+            syslog_path = '/dev/log'
+        elif os.path.exists('/var/run/syslog'):
+            syslog_path = '/var/run/syslog'
+        elif os.path.exists('/var/run/log'):
+            syslog_path = '/var/run/log'
 
     if syslog_path and not interactive_console():
         syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
         syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
         syslog_handler.setLevel(syslog_log_level)
         handlers = (console_handler, syslog_handler)
+    elif log_file:
+        file_handler = logging.handlers.WatchedFileHandler(log_file)
+        file_handler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s'))
+        file_handler.setLevel(log_file_log_level)
+        handlers = (console_handler, file_handler)
     else:
         handlers = (console_handler,)
 
-    logging.basicConfig(level=min(console_log_level, syslog_log_level), handlers=handlers)
+    logging.basicConfig(
+        level=min(console_log_level, syslog_log_level, log_file_log_level, monitoring_log_level),
+        handlers=handlers,
+    )

borgmatic/signals.py

@@ -4,8 +4,13 @@ import signal
 
 def _handle_signal(signal_number, frame):  # pragma: no cover
     '''
-    Send the signal to all processes in borgmatic's process group, which includes child process.
+    Send the signal to all processes in borgmatic's process group, which includes child processes.
     '''
+    # Prevent infinite signal handler recursion. If the parent frame is this very same handler
+    # function, we know we're recursing.
+    if frame.f_back.f_code.co_name == _handle_signal.__name__:
+        return
+
     os.killpg(os.getpgrp(), signal_number)
 
 

borgmatic/verbosity.py

@@ -1,5 +1,6 @@
 import logging
 
+VERBOSITY_ERROR = -1
 VERBOSITY_WARNING = 0
 VERBOSITY_SOME = 1
 VERBOSITY_LOTS = 2
@@ -10,6 +11,7 @@ def verbosity_to_log_level(verbosity):
     Given a borgmatic verbosity value, return the corresponding Python log level.
     '''
     return {
+        VERBOSITY_ERROR: logging.ERROR,
         VERBOSITY_WARNING: logging.WARNING,
         VERBOSITY_SOME: logging.INFO,
         VERBOSITY_LOTS: logging.DEBUG,

docs/Dockerfile

@@ -1,29 +1,32 @@
-FROM python:3.7.3-alpine3.10 as borgmatic
+FROM python:3.8-alpine3.12 as borgmatic
 
 COPY . /app
-RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
+RUN pip install --no-cache ruamel.yaml.clib==0.2.2 /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
-    && for action in init prune create check extract list info; do \
+    && for action in init prune create check extract export-tar mount umount restore list info borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:12.4.0-alpine as html
+FROM node:15.2.1-alpine as html
+
+ARG ENVIRONMENT=production
 
 WORKDIR /source
 
 RUN npm install @11ty/eleventy \
     @11ty/eleventy-plugin-syntaxhighlight \
     @11ty/eleventy-plugin-inclusive-language \
+    @11ty/eleventy-navigation \
     markdown-it \
     markdown-it-anchor \
     markdown-it-replace-link
 COPY --from=borgmatic /etc/borgmatic/config.yaml /source/docs/_includes/borgmatic/config.yaml
 COPY --from=borgmatic /command-line.txt /source/docs/_includes/borgmatic/command-line.txt
 COPY . /source
-RUN npx eleventy --input=/source/docs --output=/output/docs \
+RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.16.0-alpine
+FROM nginx:1.19.4-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/SECURITY.md

@@ -0,0 +1,19 @@
+---
+title: Security policy
+permalink: security-policy/index.html
+---
+
+## Supported versions
+
+While we want to hear about security vulnerabilities in all versions of
+borgmatic, security fixes will only be made to the most recently released
+version. It's not practical for our small volunteer effort to maintain
+multiple different release branches and put out separate security patches for
+each.
+
+## Reporting a vulnerability
+
+If you find a security vulnerability, please [file a
+ticket](https://torsion.org/borgmatic/#issues) or [send email
+directly](mailto:witten@torsion.org) as appropriate. You should expect to hear
+back within a few days at most, and generally sooner.