Monitor backups with PagerDuty hook integration (#245).

This reverts commit 24e1516ec5.

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.tox

.drone.yml

@@ -0,0 +1,127 @@
+---
+kind: pipeline
+name: python-3-5-alpine-3-10
+
+services:
+  - name: postgresql
+    image: postgres:11.6-alpine
+    environment:
+      POSTGRES_PASSWORD: test
+      POSTGRES_DB: test
+  - name: mysql
+    image: mariadb:10.3
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_DATABASE: test
+
+steps:
+- name: build
+  image: python:3.5-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-full-tests
+---
+kind: pipeline
+name: python-3-6-alpine-3-10
+
+services:
+  - name: postgresql
+    image: postgres:11.6-alpine
+    environment:
+      POSTGRES_PASSWORD: test
+      POSTGRES_DB: test
+  - name: mysql
+    image: mariadb:10.3
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_DATABASE: test
+
+steps:
+- name: build
+  image: python:3.6-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-full-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-10
+
+services:
+  - name: postgresql
+    image: postgres:11.6-alpine
+    environment:
+      POSTGRES_PASSWORD: test
+      POSTGRES_DB: test
+  - name: mysql
+    image: mariadb:10.3
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_DATABASE: test
+
+steps:
+- name: build
+  image: python:3.7-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-full-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-7
+
+services:
+  - name: postgresql
+    image: postgres:10.11-alpine
+    environment:
+      POSTGRES_PASSWORD: test
+      POSTGRES_DB: test
+  - name: mysql
+    image: mariadb:10.1
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_DATABASE: test
+
+steps:
+- name: build
+  image: python:3.7-alpine3.7
+  pull: always
+  commands:
+    - scripts/run-full-tests
+---
+kind: pipeline
+name: python-3-8-alpine-3-10
+
+services:
+  - name: postgresql
+    image: postgres:11.6-alpine
+    environment:
+      POSTGRES_PASSWORD: test
+      POSTGRES_DB: test
+  - name: mysql
+    image: mariadb:10.3
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_DATABASE: test
+
+steps:
+- name: build
+  image: python:3.8-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-full-tests
+---
+kind: pipeline
+name: documentation
+
+steps:
+- name: build
+  image: plugins/docker
+  settings:
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+    repo: witten/borgmatic-docs
+    dockerfile: docs/Dockerfile
+  when:
+    branch:
+      - master

.eleventy.js

@@ -0,0 +1,44 @@
+const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
+const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language");
+
+module.exports = function(eleventyConfig) {
+    eleventyConfig.addPlugin(pluginSyntaxHighlight);
+    eleventyConfig.addPlugin(inclusiveLangPlugin);
+
+    let markdownIt = require("markdown-it");
+    let markdownItAnchor = require("markdown-it-anchor");
+    let markdownItReplaceLink = require("markdown-it-replace-link");
+
+    let markdownItOptions = {
+        html: true,
+        breaks: false,
+        linkify: true,
+        replaceLink: function (link, env) {
+            if (process.env.NODE_ENV == "production") {
+                return link;
+            }
+            return link.replace('https://torsion.org/borgmatic/', 'http://localhost:8080/');
+        }
+    };
+    let markdownItAnchorOptions = {
+        permalink: true,
+        permalinkClass: "direct-link"
+    };
+
+    eleventyConfig.setLibrary(
+        "md",
+        markdownIt(markdownItOptions)
+            .use(markdownItAnchor, markdownItAnchorOptions)
+            .use(markdownItReplaceLink)
+    );
+
+    eleventyConfig.addPassthroughCopy({"docs/static": "static"});
+
+    return {
+        templateFormats: [
+          "md",
+          "txt"
+        ]
+    }
+};
+

.gitea/issue_template.md

@@ -0,0 +1,35 @@
+#### What I'm trying to do and why
+
+#### Steps to reproduce (if a bug)
+
+Include (sanitized) borgmatic configuration files if applicable.
+
+#### Actual behavior (if a bug)
+
+Include (sanitized) `--verbosity 2` output if applicable.
+
+#### Expected behavior (if a bug)
+
+#### Other notes / implementation ideas
+
+#### Environment
+
+**borgmatic version:** [version here]
+
+Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
+
+**borgmatic installation method:** [e.g., Debian package, Docker container, etc.]
+
+**Borg version:** [version here]
+
+Use `sudo borg --version`
+
+**Python version:** [version here]
+
+Use `python3 --version`
+
+**Database version (if applicable):** [version here]
+
+Use `psql --version` or `mysql --version` on client and server.
+
+**operating system and version:** [OS here]

.gitignore

@@ -1,6 +1,11 @@
 *.egg-info
 *.pyc
 *.swp
+.cache
+.coverage
+.pytest_cache
 .tox
-build
-dist
+__pycache__
+build/
+dist/
+pip-wheel-metadata/

.hgignore

@@ -1,9 +0,0 @@
-syntax: glob
-*.egg-info
-*.pyc
-*.swp
-.cache
-.coverage
-.tox
-build
-dist

.hgtags

@@ -1,35 +0,0 @@
-467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
-7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3
-4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4
-b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
-b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
-aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
-aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
-569aef47a9b25c55b13753f94706f5d330219995 0.0.5
-569aef47a9b25c55b13753f94706f5d330219995 0.0.5
-a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
-7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
-cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
-38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0
-ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
-ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
-7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1
-83067f995dd391e38544a7722dc3b254b59c5521 0.1.2
-acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
-6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
-6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
-e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
-0afff209b902698c2266986129d6dc9f5f913101 0.1.5
-4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6
-5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7
-977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files
-0000000000000000000000000000000000000000 github/yaml_config_files
-28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master
-dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8
-0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0
-de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
-9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2
-32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3
-5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0
-7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1
-f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2

AUTHORS

@@ -1,5 +1,14 @@
 Dan Helfman <witten@torsion.org>: Main developer
 
 Alexander Görtz: Python 3 compatibility
+Florian Lindner: Logging rewrite
 Henning Schroeder: Copy editing
-Robin `ypid` Schneider: Support additional options of Borg
+Johannes Feichtner: Support for user hooks
+Michele Lazzeri: Custom archive names
+Nick Whyte: Support prefix filtering for archive consistency checks
+newtonne: Read encryption password from external file
+Robin `ypid` Schneider: Support additional options of Borg and add validate-borgmatic-config command
+Scott Squires: Custom archive names
+Thomas LÉVEIL: Support for a keep_minutely prune option. Support for the --json option
+
+And many others! See the output of "git log".

MANIFEST.in

@@ -1 +1,2 @@
 include borgmatic/config/schema.yaml
+graft sample/systemd

NEWS

@@ -1,15 +1,498 @@
+1.5.0
+ * #245: Monitor backups with PagerDuty hook integration. See the documentation for more
+   information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook
+ * #255: Add per-action hooks: "before_prune", "after_prune", "before_check", and "after_check".
+ * #274: Add ~/.config/borgmatic.d as another configuration directory default.
+ * #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag.
+ * #280: Change "exclude_if_present" option to support multiple filenames that indicate a directory
+   should be excluded from backups, rather than just a single filename.
+ * #284: Backup to a removable drive or intermittent server via "soft failure" feature. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/
+ * #287: View consistency check progress via "--progress" flag for "check" action.
+ * For "create" and "prune" actions, no longer list files or show detailed stats at any verbosities
+   by default. You can opt back in with "--files" or "--stats" flags.
+ * For "list" and "info" actions, show repository names even at verbosity 0.
+
+1.4.22
+ * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON ouput.
+ * After a backup of a database dump in directory format, properly remove the dump directory.
+ * In "borgmatic --help", don't expand $HOME in listing of default "--config" paths.
+
+1.4.21
+ * #268: Override particular configuration options from the command-line via "--override" flag. See
+   the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
+ * #270: Only trigger "on_error" hooks and monitoring failures for "prune", "create", and "check"
+   actions, and not for other actions.
+ * When pruning with verbosity level 1, list pruned and kept archives. Previously, this information
+   was only shown at verbosity level 2.
+
+1.4.20
+ * Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option.
+ * #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and
+   "prune" actions, not just "create".
+
+1.4.19
+ * #259: Optionally change the internal database dump path via "borgmatic_source_directory" option
+   in location configuration section.
+ * #271: Support piping "borgmatic list" output to grep by logging certain log levels to console
+   stdout and others to stderr.
+ * Retain colored output when piping or redirecting in an interactive terminal.
+ * Add end-to-end tests for database dump and restore. These are run on developer machines with
+   Docker Compose for approximate parity with continuous integration tests.
+
+1.4.18
+ * Fix "--repository" flag to accept relative paths.
+ * Fix "borgmatic umount" so it only runs Borg once instead of once per repository / configuration
+   file.
+ * #253: Mount whole repositories via "borgmatic mount" without any "--archive" flag.
+ * #269: Filter listed paths via "borgmatic list --path" flag.
+
+1.4.17
+ * #235: Pass extra options directly to particular Borg commands, handy for Borg options that
+   borgmatic does not yet support natively. Use "extra_borg_options" in the storage configuration
+   section.
+ * #266: Attempt to repair any inconsistencies found during a consistency check via
+   "borgmatic check --repair" flag.
+
+1.4.16
+ * #256: Fix for "before_backup" hook not triggering an error when the command contains "borg" and
+   has an exit code of 1.
+ * #257: Fix for garbled Borg file listing when using "borgmatic create --progress" with
+   verbosity level 1 or 2.
+ * #260: Fix for missing Healthchecks monitoring payload or HTTP 500 due to incorrect unicode
+   encoding.
+
+1.4.15
+ * Fix for database dump removal incorrectly skipping some database dumps.
+ * #123: Support for mounting an archive as a FUSE filesystem via "borgmatic mount" action, and
+   unmounting via "borgmatic umount". See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#mount-a-filesystem
+
+1.4.14
+ * Show summary log errors regardless of verbosity level, and log the "summary:" header with a log
+   level based on the contained summary logs.
+
+1.4.13
+ * Show full error logs at "--verbosity 0" so you can see command output without upping the
+   verbosity level.
+
+1.4.12
+ * #247: With "borgmatic check", consider Borg warnings as errors.
+ * Dial back the display of inline error logs a bit, so failed command output doesn't appear
+   multiple times in the logs (well, except for the summary).
+
+1.4.11
+ * #241: When using the Healthchecks monitoring hook, include borgmatic logs in the payloads for
+   completion and failure pings.
+ * With --verbosity level 1 or 2, show error logs both inline when they occur and in the summary
+   logs at the bottom. With lower verbosity levels, suppress the summary and show error logs when
+   they occur.
+
+1.4.10
+ * #246: Fix for "borgmatic restore" showing success and incorrectly extracting archive files, even
+   when no databases are configured to restore. As this can overwrite files from the archive and
+   lead to data loss, please upgrade to get the fix before using "borgmatic restore".
+ * Reopen the file given by "--log-file" flag if an external program rotates the log file while
+   borgmatic is running.
+
+1.4.9
+ * #228: Database dump hooks for MySQL/MariaDB, so you can easily dump your databases before backups
+   run.
+ * #243: Fix repository does not exist error with "borgmatic extract" when repository is remote.
+
+1.4.8
+ * Monitor backups with Cronhub hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook
+ * Fix Healthchecks/Cronitor hooks to skip actions when the borgmatic "--dry-run" flag is used.
+
+1.4.7
+ * #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks.
+ * #239: Upgrade your borgmatic configuration to get new options and comments via
+   "generate-borgmatic-config --source". See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your-configuration
+
+1.4.6
+ * Verbosity level "-1" for even quieter output: Errors only (#236).
+
+1.4.5
+ * Log to file instead of syslog via command-line "--log-file" flag (#233).
+
+1.4.4
+ * #234: Support for Borg --keep-exclude-tags and --exclude-nodump options.
+
+1.4.3
+ * Monitor backups with Cronitor hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
+
+1.4.2
+ * Extract files to a particular directory via "borgmatic extract --destination" flag.
+ * Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate
+   "borgmatic restore" action. Any uses of "--restore-path" will continue working.
+
+1.4.1
+ * #229: Restore backed up PostgreSQL databases via "borgmatic restore" action. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/
+ * Documentation on how to develop borgmatic's documentation:
+   https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/#documentation-development
+
+1.4.0
+ * #225: Database dump hooks for PostgreSQL, so you can easily dump your databases before backups
+   run.
+ * #230: Rename "borgmatic list --pattern-from" flag to "--patterns-from" to match Borg.
+
+1.3.26
+ * #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
+   (non-checkpoint) archives.
+
+1.3.25
+ * #223: Dead man's switch to detect when backups start failing silently, implemented via
+   healthchecks.io hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
+ * Documentation on monitoring and alerting options for borgmatic backups:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/
+ * Automatically rewrite links when developing on documentation locally.
+
+1.3.24
+ * #86: Add "borgmatic list --successful" flag to only list successful (non-checkpoint) archives.
+ * Add a suggestion form to all documentation pages, so users can submit ideas for improving the
+   documentation.
+ * Update documentation link to community Arch Linux borgmatic package.
+
+1.3.23
+ * #174: More detailed error alerting via runtime context available in "on_error" hook.
+
+1.3.22
+ * #144: When backups to one of several repositories fails, keep backing up to the other
+   repositories and report errors afterwards.
+
+1.3.21
+ * #192: User-defined hooks for global setup or cleanup that run before/after all actions. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+
+1.3.20
+ * #205: More robust sample systemd service: boot delay, network dependency, lowered CPU/IO
+   priority, etc.
+ * #221: Fix "borgmatic create --progress" output so that it updates on the console in real-time.
+
+1.3.19
+ * #219: Fix visibility of "borgmatic prune --stats" output.
+
+1.3.18
+ * #220: Fix regression of argument parsing for default actions.
+
+1.3.17
+ * #217: Fix error with "borgmatic check --only" command-line flag with "extract" consistency check.
+
+1.3.16
+ * #210: Support for Borg check --verify-data flag via borgmatic "data" consistency check.
+ * #210: Override configured consistency checks via "borgmatic check --only" command-line flag.
+ * When generating sample configuration with generate-borgmatic-config, add a space after each "#"
+   comment indicator.
+
+1.3.15
+ * #208: Fix for traceback when the "checks" option has an empty value.
+ * #209: Bypass Borg error about a moved repository via "relocated_repo_access_is_ok" option in
+   borgmatic storage configuration section.
+ * #213: Reorder arguments passed to Borg to fix duplicate directories when using Borg patterns.
+ * #214: Fix for hook erroring with exit code 1 not being interpreted as an error.
+
+1.3.14
+ * #204: Do not treat Borg warnings (exit code 1) as failures.
+ * When validating configuration files, require strings instead of allowing any scalar type.
+
+1.3.13
+ * #199: Add note to documentation about using spaces instead of tabs for indentation, as YAML does
+   not allow tabs.
+ * #203: Fix compatibility with ruamel.yaml 0.16.x.
+ * If a "prefix" option in borgmatic's configuration has an empty value (blank or ""), then disable
+   default prefix.
+
+1.3.12
+ * Only log to syslog when run from a non-interactive console (e.g. a cron job).
+ * Remove unicode byte order mark from syslog output so it doesn't show up as a literal in rsyslog
+   output. See discussion on #197.
+
+1.3.11
+ * #193: Pass through several "borg list" and "borg info" flags like --short, --format, --sort-by,
+   --first, --last, etc. via borgmatic command-line flags.
+ * Add borgmatic info --repository and --archive command-line flags to display info for individual
+   repositories or archives.
+ * Support for Borg --noatime, --noctime, and --nobirthtime flags via corresponding options in
+   borgmatic configuration location section.
+
+1.3.10
+ * #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero.
+
+1.3.9
+ * #195: Switch to command-line actions as more traditional sub-commands, e.g. "borgmatic create",
+   "borgmatic prune", etc. However, the classic dashed options like "--create" still work!
+
+1.3.8
+ * #191: Disable console color via "color" option in borgmatic configuration output section.
+
+1.3.7
+ * #196: Fix for unclear error message for invalid YAML merge include.
+ * #197: Don't color syslog output.
+ * Change default syslog verbosity to show errors only.
+
+1.3.6
+ * #53: Log to syslog in addition to existing console logging. Add --syslog-verbosity flag to
+   customize the log level. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/
+ * #178: Look for .yml configuration file extension in addition to .yaml.
+ * #189: Set umask used when executing hooks via "umask" option in borgmatic hooks section.
+ * Remove Python cache files before each Tox run.
+ * Add #borgmatic Freenode IRC channel to documentation.
+ * Add Borg/borgmatic hosting providers section to documentation.
+ * Add files for building documentation into a Docker image for web serving.
+ * Upgrade project build server from Drone 0.8 to 1.1.
+ * Build borgmatic documentation during continuous integration.
+ * We're nearly at 500 ★s on GitHub. We can do this!
+
+1.3.5
+ * #153: Support for various Borg directory environment variables (BORG_CONFIG_DIR, BORG_CACHE_DIR,
+   etc.) via options in borgmatic's storage configuration.
+ * #177: Fix for regression with missing verbose log entries.
+
+1.3.4
+ * Part of #125: Color borgmatic (but not Borg) output when using an interactive terminal.
+ * #166: Run tests for all installed versions of Python.
+ * #168: Update README with continuous integration badge.
+ * #169: Automatically sort Python imports in code.
+ * Document installing borgmatic with pip install --user instead of a system Python install.
+ * Get more reproducible builds by pinning the versions of pip and tox used to run tests.
+ * Factor out build/test configuration from tox.ini file.
+
+1.3.3
+ * Add validate-borgmatic-config command, useful for validating borgmatic config generated by
+   configuration management or even edited by hand.
+
+1.3.2
+ * #160: Fix for hooks executing when using --dry-run. Now hooks are skipped during a dry run.
+
+1.3.1
+ * #155: Fix for invalid JSON output when using multiple borgmatic configuration files.
+ * #157: Fix for seemingly random filename ordering when running through a directory of
+   configuration files.
+ * Fix for empty JSON output when using --create --json.
+ * Now capturing Borg output only when --json flag is used. Previously, borgmatic delayed Borg
+   output even without the --json flag.
+
+1.3.0
+ * #148: Configuration file includes and merging via "!include" tag to support reuse of common
+   options across configuration files.
+
+1.2.18
+ * #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
+   borgmatic's location section.
+
+1.2.17
+ * #140: List the files within an archive via --list --archive option.
+
+1.2.16
+ * #119: Include a sample borgmatic configuration file in the documentation.
+ * #123: Support for Borg archive restoration via borgmatic --extract command-line flag.
+ * Refactor documentation into multiple separate pages for clarity and findability.
+ * Organize options within command-line help into logical groups.
+ * Exclude tests from distribution packages.
+
+1.2.15
+ * #127: Remove date echo from schema example, as it's not a substitute for real logging.
+ * #132: Leave exclude_patterns glob expansion to Borg, since doing it in borgmatic leads to
+   confusing behavior.
+ * #136: Handle and format validation errors raised during argument parsing.
+ * #138: Allow use of --stats flag when --create or --prune flags are implied.
+
+1.2.14
+ * #103: When generating sample configuration with generate-borgmatic-config, document the defaults
+   for each option.
+ * #116: When running multiple configuration files, attempt all configuration files even if one of
+   them errors. Log a summary of results at the end.
+ * Add borgmatic --version command-line flag to get the current installed version number.
+
+1.2.13
+ * #100: Support for --stats command-line flag independent of --verbosity.
+ * #117: With borgmatic --init command-line flag, proceed without erroring if a repository already
+   exists.
+
+1.2.12
+ * #110: Support for Borg repository initialization via borgmatic --init command-line flag.
+ * #111: Update Borg create --filter values so a dry run lists files to back up.
+ * #113: Update README with link to a new/forked Docker image.
+ * Prevent deprecated --excludes command-line option from being used.
+ * Refactor README a bit to flow better for first-time users.
+ * Update README with a few additional borgmatic packages (Debian and Ubuntu).
+
+1.2.11
+ * #108: Support for Borg create --progress via borgmatic command-line flag.
+
+1.2.10
+ * #105: Support for Borg --chunker-params create option via "chunker_params" option in borgmatic's
+   storage section.
+
+1.2.9
+ * #102: Fix for syntax error that occurred in Python 3.5 and below.
+ * Make automated tests support running in Python 3.5.
+
+1.2.8
+ * #73: Enable consistency checks for only certain repositories via "check_repositories" option in
+   borgmatic's consistency configuration. Handy for large repositories that take forever to check.
+ * Include link to issue tracker within various command output.
+ * Run continuous integration tests on a matrix of Python and Borg versions.
+
+1.2.7
+ * #98: Support for Borg --keep-secondly prune option.
+ * Use Black code formatter and Flake8 code checker as part of running automated tests.
+ * Add an end-to-end automated test that actually integrates with Borg.
+ * Set up continuous integration for borgmatic automated tests on projects.evoworx.org.
+
+1.2.6
+ * Fix generated configuration to also include a "keep_daily" value so pruning works out of the
+   box.
+
+1.2.5
+ * #57: When generating sample configuration with generate-borgmatic-config, comment out all
+   optional configuration so as to streamline the initial configuration process.
+
+1.2.4
+ * Fix for archive checking traceback due to parameter mismatch.
+
+1.2.3
+ * #64, #90, #92: Rewrite of logging system. Now verbosity flags passed to Borg are derived from
+   borgmatic's log level. Note that the output of borgmatic might slightly change.
+ * Part of #80: Support for Borg create --read-special via "read_special" option in borgmatic's
+   location configuration.
+ * #87: Support for Borg create --checkpoint-interval via "checkpoint_interval" option in
+   borgmatic's storage configuration.
+ * #88: Fix declared pykwalify compatibility version range in setup.py to prevent use of ancient
+   versions of pykwalify with large version numbers.
+ * #89: Pass --show-rc option to Borg when at highest verbosity level.
+ * #94: Support for Borg --json option via borgmatic command-line to --create archives.
+
+1.2.2
+ * #85: Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52, which manifested in
+   borgmatic as a pykwalify RuleError.
+
+1.2.1
+ * Skip before/after backup hooks when only doing --prune, --check, --list, and/or --info.
+ * #71: Support for XDG_CONFIG_HOME environment variable for specifying alternate user ~/.config/
+   path.
+ * #74, #83: Support for Borg --json option via borgmatic command-line to --list archives or show
+   archive --info in JSON format, ideal for programmatic consumption.
+ * #38, #76: Upgrade ruamel.yaml compatibility version range and fix support for Python 3.7.
+ * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files,
+   editor swap files, etc.
+ * #81: Document user-defined hooks run before/after backup, or on error.
+ * Add code style guidelines to the documention.
+
+1.2.0
+ * #61: Support for Borg --list option via borgmatic command-line to list all archives.
+ * #61: Support for Borg --info option via borgmatic command-line to display summary information.
+ * #62: Update README to mention other ways of installing borgmatic.
+ * Support for Borg --prefix option for consistency checks via "prefix" option in borgmatic's
+   consistency configuration.
+ * Add introductory screencast link to documentation.
+ * #59: Ignore "check_last" and consistency "prefix" when "archives" not in consistency checks.
+ * #60: Add "Persistent" flag to systemd timer example.
+ * #63: Support for Borg --nobsdflags option to skip recording bsdflags (e.g. NODUMP, IMMUTABLE) in
+   archive.
+ * #69: Support for Borg prune --umask option using value of existing "umask" option in borgmatic's
+   storage configuration.
+ * Update tox.ini to only assume Python 3.x instead of Python 3.4 specifically.
+ * Add ~/.config/borgmatic/config.yaml to default configuration path probing.
+ * Document how to develop on and contribute to borgmatic.
+
+1.1.15
+ * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
+ * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
+   Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298
+ * #55: Fix for missing tags/releases on Gitea and GitHub project hosting.
+ * #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock.
+ * #58: Support for using tilde in exclude_patterns to reference home directory.
+
+1.1.14
+ * #49: Fix for typo in --patterns-from option.
+ * #47: Support for Borg --dry-run option via borgmatic command-line.
+
+1.1.13
+ * #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository",
+   "archives", and "extract") are specified in borgmatic configuration.
+ * #48: Add "local_path" to configuration for specifying an alternative Borg executable path.
+ * #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed
+   includes/excludes.
+ * Moved issue tracker from Taiga to integrated Gitea tracker at
+   https://projects.torsion.org/witten/borgmatic/issues
+
+1.1.12
+ * #46: Declare dependency on pykwalify 1.6 or above, as older versions yield "Unknown key: version"
+   rule errors.
+ * Support for Borg --keep-minutely prune option.
+
+1.1.11
+ * #26: Add "ssh_command" to configuration for specifying a custom SSH command or options.
+ * Fix for incorrect /etc/borgmatic.d/ configuration path probing on macOS. This problem manifested
+   as an error on startup: "[Errno 2] No such file or directory: '/etc/borgmatic.d'".
+
+1.1.10
+ * Pass several Unix signals through to child processes like Borg. This means that Borg now properly
+   shuts down if borgmatic is terminated (e.g. due to a system suspend).
+ * #30: Support for using tilde in repository paths to reference home directory.
+ * #43: Support for Borg --files-cache option for setting the files cache operation mode.
+ * #45: Support for Borg --remote-ratelimit option for limiting upload rate.
+ * Log invoked Borg commands when at highest verbosity level.
+
+1.1.9
+ * #17, #39: Support for user-defined hooks before/after backup, or on error.
+ * #34: Improve clarity of logging spew at high verbosity levels.
+ * #30: Support for using tilde in source directory path to reference home directory.
+ * Require "prefix" in retention section when "archive_name_format" is set. This is to avoid
+   accidental pruning of archives with a different archive name format. For similar reasons, default
+   "prefix" to "{hostname}-" if not specified.
+ * Convert main source repository from Mercurial to Git.
+ * Update dead links to Borg documentation.
+
+1.1.8
+ * #40: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default
+   config paths.
+
+1.1.7
+
+ * #29: Add "archive_name_format" to configuration for customizing archive names.
+ * Fix for traceback when "exclude_from" value is empty in configuration file.
+ * When pruning, make highest verbosity level list archives kept and pruned.
+ * Clarification of Python 3 pip usage in documentation.
+
+1.1.6
+
+ * #13, #36: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.
+
+1.1.5
+
+ * #35: New "extract" consistency check that performs a dry-run extraction of the most recent
+   archive.
+
+1.1.4
+
+ * #18: Added command-line flags for performing a borgmatic run with only pruning, creating, or
+   checking enabled. This supports use cases like running consistency checks from a different cron
+   job with a different frequency, or running pruning with a different verbosity level.
+
 1.1.3
 
- * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
+ * #15: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
  * Fix for generate-borgmatic-config writing config with invalid one_file_system value.
 
 1.1.2
 
- * #32: Fix for passing check_last as integer to subprocess when calling Borg.
+ * #33: Fix for passing check_last as integer to subprocess when calling Borg.
 
 1.1.1
 
- * Part of #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of
+ * Part of #33: Fix for upgrade-borgmatic-config converting check_last option as a string instead of
    an integer.
  * Fix for upgrade-borgmatic-config erroring when consistency checks option is not present.
 
@@ -18,8 +501,8 @@
  * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade.
  * Added generate-borgmatic-config command for initial config creation.
  * Dropped Python 2 support. Now Python 3 only.
- * #18: Fix for README mention of sample files not included in package.
- * #22: Sample files for triggering borgmatic from a systemd timer.
+ * #19: Fix for README mention of sample files not included in package.
+ * #23: Sample files for triggering borgmatic from a systemd timer.
  * Support for backing up to multiple repositories.
  * To free up space, now pruning backups prior to creating a new backup.
  * Enabled test coverage output during tox runs.
@@ -27,15 +510,15 @@
 
 1.0.3
 
- * #21: Fix for verbosity flag not actually causing verbose output.
+ * #22: Fix for verbosity flag not actually causing verbose output.
 
 1.0.2
 
- * #20: Fix for traceback when remote_path option is missing.
+ * #21: Fix for traceback when remote_path option is missing.
 
 1.0.1
 
- * #19: Support for Borg's --remote-path option to use an alternate Borg
+ * #20: Support for Borg's --remote-path option to use an alternate Borg
    executable. See sample/config.
 
 1.0.0
@@ -57,13 +540,13 @@
 
 0.1.7
 
- * #11: Fixed parsing of punctuation in configuration file.
+ * #12: Fixed parsing of punctuation in configuration file.
  * Better error message when configuration file is missing.
 
 0.1.6
 
- * #9: New configuration option for the encryption passphrase.
- * #10: Support for Borg's new archive compression feature.
+ * #10: New configuration option for the encryption passphrase.
+ * #11: Support for Borg's new archive compression feature.
 
 0.1.5
 
@@ -75,7 +558,7 @@
 
 0.1.3
 
- * #1: Add support for "borg check --last N" to Borg backend.
+ * #2: Add support for "borg check --last N" to Borg backend.
 
 0.1.2
 

README.md

@@ -1,276 +1,142 @@
+---
 title: borgmatic
+permalink: index.html
+---
 
-<img src="static/borgmatic.svg" alt="borgmatic logo" style="width: 8em; float: right; padding-left: 1em;" />
+## It's your data. Keep it that way.
 
-## Overview
+<img src="docs/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
 
-borgmatic (formerly atticmatic) is a simple Python wrapper script for the
-[Borg](https://borgbackup.readthedocs.org/en/stable/) backup software that
-initiates a backup, prunes any old backups according to a retention policy,
-and validates backups for consistency. The script supports specifying your
-settings in a declarative configuration file rather than having to put them
-all on the command-line, and handles common errors.
+borgmatic is simple, configuration-driven backup software for servers and
+workstations. Protect your files with client-side encryption. Backup your
+databases too. Monitor it all with integrated third-party services.
 
-Here's an example config file:
+Here's an example configuration file:
 
 ```yaml
 location:
-    # List of source directories to backup. Globs are expanded.
+    # List of source directories to backup.
     source_directories:
         - /home
         - /etc
-        - /var/log/syslog*
 
-    # Paths to local or remote repositories.
+    # Paths of local or remote repositories to backup to.
     repositories:
-        - user@backupserver:sourcehostname.borg
-
-    # Any paths matching these patterns are excluded from backups.
-    exclude_patterns:
-        - /home/*/.cache
+        - 1234@usw-s001.rsync.net:backups.borg
+        - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
+        - /var/lib/backups/local.borg
 
 retention:
-    # Retention policy for how many backups to keep in each category.
+    # Retention policy for how many backups to keep.
     keep_daily: 7
     keep_weekly: 4
     keep_monthly: 6
 
 consistency:
-    # List of consistency checks to run: "repository", "archives", or both.
+    # List of checks to run to validate your backups.
     checks:
         - repository
         - archives
+
+hooks:
+    # Custom preparation scripts to run.
+    before_backup:
+        - prepare-for-backup.sh
+
+    # Databases to dump and include in backups.
+    postgresql_databases:
+        - name: users
+
+    # Third-party services to notify you if backups aren't happening.
+    healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
 ```
 
-borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
-available](https://torsion.org/hg/borgmatic). It's also mirrored on
-[GitHub](https://github.com/witten/borgmatic) and
-[BitBucket](https://bitbucket.org/dhelfman/borgmatic) for convenience.
+Want to see borgmatic in action? Check out the <a
+href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
 
+<script src="https://asciinema.org/a/203761.js" id="asciicast-203761" async></script>
 
-## Installation
+borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 
-To get up and running, follow the [Borg Quick
-Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
-a repository on a local or remote host. Note that if you plan to run
-borgmatic on a schedule with cron, and you encrypt your Borg repository with
-a passphrase instead of a key file, you'll need to set the borgmatic
-`encryption_passphrase` configuration variable. See the repository encryption
-section of the Quick Start for more info.
+## Integrations
 
-If the repository is on a remote host, make sure that your local root user has
-key-based ssh access to the desired user account on the remote host.
+<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic"><img src="docs/static/rsyncnet.png" alt="rsync.net" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 
-To install borgmatic, run the following command to download and install it:
 
-    sudo pip install --upgrade borgmatic
+## How-to guides
 
-Make sure you're using Python 3, as borgmatic does not support Python 2. (You
-may have to use "pip3" or similar instead of "pip".)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/) ⬅ *Start here!*
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+ * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+ * [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/)
+ * [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Backup to a removable drive or an intermittent server](https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/)
+ * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
 
-## Configuration
 
-After you install borgmatic, generate a sample configuration file:
+## Reference guides
 
-    sudo generate-borgmatic-config
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)
 
-This generates a sample configuration file at /etc/borgmatic/config.yaml (by
-default). You should edit the file to suit your needs, as the values are just
-representative. All fields are optional except where indicated, so feel free
-to remove anything you don't need.
 
+## Hosting providers
 
-### Multiple configuration files
+Need somewhere to store your encrypted offsite backups? The following hosting
+providers include specific support for Borg/borgmatic. Using these links and
+services helps support borgmatic development and hosting. (These are referral
+links, but without any tracking scripts or cookies.)
 
-A more advanced usage is to create multiple separate configuration files and
-place each one in a /etc/borgmatic.d directory. For instance:
+<ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+</ul>
 
-    sudo mkdir /etc/borgmatic.d
-    sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
-    sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
+## Support and contributing
 
-With this approach, you can have entirely different backup policies for
-different applications on your system. For instance, you may want one backup
-configuration for your database data directory, and a different configuration
-for your user home directories.
+### Issues
 
-When you set up multiple configuration files like this, borgmatic will run
-each one in turn from a single borgmatic invocation. This includes, by
-default, the traditional /etc/borgmatic/config.yaml as well.
+You've got issues? Or an idea for a feature enhancement? We've got an [issue
+tracker](https://projects.torsion.org/witten/borgmatic/issues). In order to
+create a new issue or comment on an issue, you'll need to [login
+first](https://projects.torsion.org/user/login). Note that you can login with
+an existing GitHub account if you prefer.
 
-
-## Upgrading
-
-In general, all you should need to do to upgrade borgmatic is run the
-following:
-
-    sudo pip install --upgrade borgmatic
-
-(You may have to use "pip3" or similar instead of "pip", so Python 3 gets
-used.)
-
-However, see below about special cases.
-
-
-### Upgrading from borgmatic 1.0.x
-
-borgmatic changed its configuration file format in version 1.1.0 from
-INI-style to YAML. This better supports validation, and has a more natural way
-to express lists of values. To upgrade your existing configuration, first
-upgrade to the new version of borgmatic:
-
-As of version 1.1.0, borgmatic no longer supports Python 2. If you were
-already running borgmatic with Python 3, then you can simply upgrade borgmatic
-in-place:
-
-    sudo pip install --upgrade borgmatic
-
-But if you were running borgmatic with Python 2, uninstall and reinstall instead:
-
-    sudo pip uninstall borgmatic
-    sudo pip3 install borgmatic
-
-The pip binary names for different versions of Python can differ, so the above
-commands may need some tweaking to work on your machine.
-
-
-Once borgmatic is upgraded, run:
-
-    sudo upgrade-borgmatic-config
-
-That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
-(by default) using the values from both your existing configuration and
-excludes files. The new version of borgmatic will consume the YAML
-configuration file instead of the old one.
-
-
-### Upgrading from atticmatic
-
-You can ignore this section if you're not an atticmatic user (the former name
-of borgmatic).
-
-borgmatic only supports Borg now and no longer supports Attic. So if you're
-an Attic user, consider switching to Borg. See the [Borg upgrade
-command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
-for more information. Then, follow the instructions above about setting up
-your borgmatic configuration files.
-
-If you were already using Borg with atticmatic, then you can easily upgrade
-from atticmatic to borgmatic. Simply run the following commands:
-
-    sudo pip uninstall atticmatic
-    sudo pip install borgmatic
-
-That's it! borgmatic will continue using your /etc/borgmatic configuration
-files.
-
-
-## Usage
-
-You can run borgmatic and start a backup simply by invoking it without
-arguments:
-
-    borgmatic
-
-This will also prune any old backups as per the configured retention policy,
-and check backups for consistency problems due to things like file damage.
-
-By default, the backup will proceed silently except in the case of errors. But
-if you'd like to to get additional information about the progress of the
-backup as it proceeds, use the verbosity option:
-
-    borgmatic --verbosity 1
-
-Or, for even more progress spew:
-
-    borgmatic --verbosity 2
-
-If you'd like to see the available command-line arguments, view the help:
-
-    borgmatic --help
-
-
-## Autopilot
-
-If you want to run borgmatic automatically, say once a day, the you can
-configure a job runner to invoke it periodically.
-
-### cron
-
-If you're using cron, download the [sample cron
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/cron/borgmatic).
-Then, from the directory where you downloaded it:
-
-    sudo mv borgmatic /etc/cron.d/borgmatic
-    sudo chmod +x /etc/cron.d/borgmatic
-
-You can modify the cron file if you'd like to run borgmatic more or less frequently.
-
-### systemd
-
-If you're using systemd instead of cron to run jobs, download the [sample
-systemd service
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.service)
-and the [sample systemd timer
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.timer).
-Then, from the directory where you downloaded them:
-
-    sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
-    sudo systemctl enable borgmatic.timer
-    sudo systemctl start borgmatic.timer
-
-Feel free to modify the timer file based on how frequently you'd like
-borgmatic to run.
-
-
-## Running tests
-
-First install tox, which is used for setting up testing environments:
-
-    pip install tox
-
-Then, to actually run tests, run:
-
-    tox
-
-
-## Troubleshooting
-
-### Broken pipe with remote repository
-
-When running borgmatic on a large remote repository, you may receive errors
-like the following, particularly while "borg check" is validating backups for
-consistency:
-
-    Write failed: Broken pipe
-    borg: Error: Connection closed by remote host
-
-This error can be caused by an ssh timeout, which you can rectify by adding
-the following to the ~/.ssh/config file on the client:
-
-    Host *
-        ServerAliveInterval 120
-
-This should make the client keep the connection alive while validating
-backups.
-
-
-### libyaml compilation errors
-
-borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
-use a C YAML library (libyaml) if present. But if it's not installed, then
-when installing or upgrading borgmatic, you may see errors about compiling the
-YAML library. If so, not to worry. borgmatic should install and function
-correctly even without the C YAML library. And borgmatic won't be any faster
-with the C library present, so you don't need to go out of your way to install
-it.
-
-
-## Issues and feedback
-
-Got an issue or an idea for a feature enhancement? Check out the [borgmatic
-issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues?page=1&status=399951,399952,399955). In
-order to create a new issue or comment on an issue, you'll need to [login
-first](https://tree.taiga.io/login).
+If you'd like to chat with borgmatic developers or users, head on over to the
+`#borgmatic` IRC channel on Freenode, either via <a
+href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
+native <a href="irc://chat.freenode.net:6697">IRC client</a>.
 
 Other questions or comments? Contact <mailto:witten@torsion.org>.
+
+
+### Contributing
+
+borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
+available](https://projects.torsion.org/witten/borgmatic). It's also mirrored
+on [GitHub](https://github.com/witten/borgmatic) for convenience.
+
+If you'd like to contribute to borgmatic development, please feel free to
+submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls)
+or open an [issue](https://projects.torsion.org/witten/borgmatic/issues) first
+to discuss your idea. We also accept Pull Requests on GitHub, if that's more
+your thing. In general, contributions are very welcome. We don't bite! 
+
+Also, please check out the [borgmatic development
+how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
+info on cloning source code, running tests, etc.
+
+<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master)</a>
+

borgmatic/borg.py

@@ -1,202 +0,0 @@
-from datetime import datetime
-import glob
-import itertools
-import os
-import platform
-import re
-import subprocess
-import tempfile
-
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-# Integration with Borg for actually handling backups.
-
-
-COMMAND = 'borg'
-
-
-def initialize(storage_config, command=COMMAND):
-    passphrase = storage_config.get('encryption_passphrase')
-
-    if passphrase:
-        os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase
-
-
-def _write_exclude_file(exclude_patterns=None):
-    '''
-    Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
-    None if no patterns are provided.
-    '''
-    if not exclude_patterns:
-        return None
-
-    exclude_file = tempfile.NamedTemporaryFile('w')
-    exclude_file.write('\n'.join(exclude_patterns))
-    exclude_file.flush()
-
-    return exclude_file
-
-
-def create_archive(
-    verbosity, repository, location_config, storage_config, command=COMMAND,
-):
-    '''
-    Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
-    repository path, a list of exclude patterns, and a command to run, create an attic archive.
-    '''
-    sources = tuple(
-        itertools.chain.from_iterable(
-            glob.glob(directory) or [directory]
-            for directory in location_config['source_directories']
-        )
-    )
-
-    exclude_file = _write_exclude_file(location_config.get('exclude_patterns'))
-    exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
-    compression = storage_config.get('compression', None)
-    compression_flags = ('--compression', compression) if compression else ()
-    umask = storage_config.get('umask', None)
-    umask_flags = ('--umask', str(umask)) if umask else ()
-    one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
-    remote_path = location_config.get('remote_path')
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--list', '--stats'),
-    }.get(verbosity, ())
-
-    full_command = (
-        command, 'create',
-        '{repo}::{hostname}-{timestamp}'.format(
-            repo=repository,
-            hostname=platform.node(),
-            timestamp=datetime.now().isoformat(),
-        ),
-    ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
-        remote_path_flags + umask_flags + verbosity_flags
-
-    subprocess.check_call(full_command)
-
-
-def _make_prune_flags(retention_config):
-    '''
-    Given a retention config dict mapping from option name to value, tranform it into an iterable of
-    command-line name-value flag pairs.
-
-    For example, given a retention config of:
-
-        {'keep_weekly': 4, 'keep_monthly': 6}
-
-    This will be returned as an iterable of:
-
-        (
-            ('--keep-weekly', '4'),
-            ('--keep-monthly', '6'),
-        )
-    '''
-    return (
-        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
-        for option_name, value in retention_config.items()
-    )
-
-
-def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None):
-    '''
-    Given a verbosity flag, a local or remote repository path, a retention config dict, and a
-    command to run, prune attic archives according the the retention policy specified in that
-    configuration.
-    '''
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--stats'),
-    }.get(verbosity, ())
-
-    full_command = (
-        command, 'prune',
-        repository,
-    ) + tuple(
-        element
-        for pair in _make_prune_flags(retention_config)
-        for element in pair
-    ) + remote_path_flags + verbosity_flags
-
-    subprocess.check_call(full_command)
-
-
-DEFAULT_CHECKS = ('repository', 'archives')
-
-
-def _parse_checks(consistency_config):
-    '''
-    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
-
-    For example, given a retention config of:
-
-        {'checks': ['repository', 'archives']}
-
-    This will be returned as:
-
-        ('repository', 'archives')
-
-    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
-    "disabled", return an empty tuple, meaning that no checks should be run.
-    '''
-    checks = consistency_config.get('checks', [])
-    if checks == ['disabled']:
-        return ()
-
-    return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
-
-
-def _make_check_flags(checks, check_last=None):
-    '''
-    Given a parsed sequence of checks, transform it into tuple of command-line flags.
-
-    For example, given parsed checks of:
-
-        ('repository',)
-
-    This will be returned as:
-
-        ('--repository-only',)
-
-    Additionally, if a check_last value is given, a "--last" flag will be added.
-    '''
-    last_flag = ('--last', str(check_last)) if check_last else ()
-    if checks == DEFAULT_CHECKS:
-        return last_flag
-
-    return tuple(
-        '--{}-only'.format(check) for check in checks
-    ) + last_flag
-
-
-def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None):
-    '''
-    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
-    command to run, check the contained attic archives for consistency.
-
-    If there are no consistency checks to run, skip running them.
-    '''
-    checks = _parse_checks(consistency_config)
-    check_last = consistency_config.get('check_last', None)
-    if not checks:
-        return
-
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info',),
-        VERBOSITY_LOTS: ('--debug',),
-    }.get(verbosity, ())
-
-    full_command = (
-        command, 'check',
-        repository,
-    ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
-
-    # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
-    stdout = None if verbosity_flags else open(os.devnull, 'w')
-
-    subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)

borgmatic/borg/check.py

@@ -0,0 +1,142 @@
+import logging
+
+from borgmatic.borg import extract
+from borgmatic.execute import execute_command, execute_command_without_capture
+
+DEFAULT_CHECKS = ('repository', 'archives')
+DEFAULT_PREFIX = '{hostname}-'
+
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_checks(consistency_config, only_checks=None):
+    '''
+    Given a consistency config with a "checks" list, and an optional list of override checks,
+    transform them a tuple of named checks to run.
+
+    For example, given a retention config of:
+
+        {'checks': ['repository', 'archives']}
+
+    This will be returned as:
+
+        ('repository', 'archives')
+
+    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If the checks value
+    is the string "disabled", return an empty tuple, meaning that no checks should be run.
+
+    If the "data" option is present, then make sure the "archives" option is included as well.
+    '''
+    checks = [
+        check.lower() for check in (only_checks or consistency_config.get('checks', []) or [])
+    ]
+    if checks == ['disabled']:
+        return ()
+
+    if 'data' in checks and 'archives' not in checks:
+        checks.append('archives')
+
+    return tuple(check for check in checks if check not in ('disabled', '')) or DEFAULT_CHECKS
+
+
+def _make_check_flags(checks, check_last=None, prefix=None):
+    '''
+    Given a parsed sequence of checks, transform it into tuple of command-line flags.
+
+    For example, given parsed checks of:
+
+        ('repository',)
+
+    This will be returned as:
+
+        ('--repository-only',)
+
+    However, if both "repository" and "archives" are in checks, then omit them from the returned
+    flags because Borg does both checks by default.
+
+    Additionally, if a check_last value is given and "archives" is in checks, then include a
+    "--last" flag. And if a prefix value is given and "archives" is in checks, then include a
+    "--prefix" flag.
+    '''
+    if 'archives' in checks:
+        last_flags = ('--last', str(check_last)) if check_last else ()
+        prefix_flags = ('--prefix', prefix) if prefix else ()
+    else:
+        last_flags = ()
+        prefix_flags = ()
+        if check_last:
+            logger.warning(
+                'Ignoring check_last option, as "archives" is not in consistency checks.'
+            )
+        if prefix:
+            logger.warning(
+                'Ignoring consistency prefix option, as "archives" is not in consistency checks.'
+            )
+
+    common_flags = last_flags + prefix_flags + (('--verify-data',) if 'data' in checks else ())
+
+    if set(DEFAULT_CHECKS).issubset(set(checks)):
+        return common_flags
+
+    return (
+        tuple('--{}-only'.format(check) for check in checks if check in DEFAULT_CHECKS)
+        + common_flags
+    )
+
+
+def check_archives(
+    repository,
+    storage_config,
+    consistency_config,
+    local_path='borg',
+    remote_path=None,
+    progress=None,
+    repair=None,
+    only_checks=None,
+):
+    '''
+    Given a local or remote repository path, a storage config dict, a consistency config dict,
+    local/remote commands to run, whether to include progress information, whether to attempt a
+    repair, and an optional list of checks to use instead of configured checks, check the contained
+    Borg archives for consistency.
+
+    If there are no consistency checks to run, skip running them.
+    '''
+    checks = _parse_checks(consistency_config, only_checks)
+    check_last = consistency_config.get('check_last', None)
+    lock_wait = None
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
+
+    if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
+        lock_wait = storage_config.get('lock_wait', None)
+
+        verbosity_flags = ()
+        if logger.isEnabledFor(logging.INFO):
+            verbosity_flags = ('--info',)
+        if logger.isEnabledFor(logging.DEBUG):
+            verbosity_flags = ('--debug', '--show-rc')
+
+        prefix = consistency_config.get('prefix', DEFAULT_PREFIX)
+
+        full_command = (
+            (local_path, 'check')
+            + (('--repair',) if repair else ())
+            + _make_check_flags(checks, check_last, prefix)
+            + (('--remote-path', remote_path) if remote_path else ())
+            + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+            + verbosity_flags
+            + (('--progress',) if progress else ())
+            + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+            + (repository,)
+        )
+
+        # The Borg repair option trigger an interactive prompt, which won't work when output is
+        # captured. And progress messes with the terminal directly.
+        if repair or progress:
+            execute_command_without_capture(full_command, error_on_warnings=True)
+        else:
+            execute_command(full_command, error_on_warnings=True)
+
+    if 'extract' in checks:
+        extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -0,0 +1,212 @@
+import glob
+import itertools
+import logging
+import os
+import tempfile
+
+from borgmatic.execute import execute_command, execute_command_without_capture
+
+logger = logging.getLogger(__name__)
+
+
+def _expand_directory(directory):
+    '''
+    Given a directory path, expand any tilde (representing a user's home directory) and any globs
+    therein. Return a list of one or more resulting paths.
+    '''
+    expanded_directory = os.path.expanduser(directory)
+
+    return glob.glob(expanded_directory) or [expanded_directory]
+
+
+def _expand_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes and globs in each one. Return all the
+    resulting directories as a single flattened tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(_expand_directory(directory) for directory in directories)
+    )
+
+
+def _expand_home_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
+    Return the results as a tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(os.path.expanduser(directory) for directory in directories)
+
+
+def _write_pattern_file(patterns=None):
+    '''
+    Given a sequence of patterns, write them to a named temporary file and return it. Return None
+    if no patterns are provided.
+    '''
+    if not patterns:
+        return None
+
+    pattern_file = tempfile.NamedTemporaryFile('w')
+    pattern_file.write('\n'.join(patterns))
+    pattern_file.flush()
+
+    return pattern_file
+
+
+def _make_pattern_flags(location_config, pattern_filename=None):
+    '''
+    Given a location config dict with a potential patterns_from option, and a filename containing
+    any additional patterns, return the corresponding Borg flags for those files as a tuple.
+    '''
+    pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
+        (pattern_filename,) if pattern_filename else ()
+    )
+
+    return tuple(
+        itertools.chain.from_iterable(
+            ('--patterns-from', pattern_filename) for pattern_filename in pattern_filenames
+        )
+    )
+
+
+def _make_exclude_flags(location_config, exclude_filename=None):
+    '''
+    Given a location config dict with various exclude options, and a filename containing any exclude
+    patterns, return the corresponding Borg flags as a tuple.
+    '''
+    exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
+        (exclude_filename,) if exclude_filename else ()
+    )
+    exclude_from_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-from', exclude_filename) for exclude_filename in exclude_filenames
+        )
+    )
+    caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
+    if_present_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-if-present', if_present)
+            for if_present in location_config.get('exclude_if_present', ())
+        )
+    )
+    keep_exclude_tags_flags = (
+        ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else ()
+    )
+    exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else ()
+
+    return (
+        exclude_from_flags
+        + caches_flag
+        + if_present_flags
+        + keep_exclude_tags_flags
+        + exclude_nodump_flags
+    )
+
+
+DEFAULT_BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
+
+
+def borgmatic_source_directories(borgmatic_source_directory):
+    '''
+    Return a list of borgmatic-specific source directories used for state like database backups.
+    '''
+    if not borgmatic_source_directory:
+        borgmatic_source_directory = DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+
+    return (
+        [borgmatic_source_directory]
+        if os.path.exists(os.path.expanduser(borgmatic_source_directory))
+        else []
+    )
+
+
+def create_archive(
+    dry_run,
+    repository,
+    location_config,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+    progress=False,
+    stats=False,
+    json=False,
+    files=False,
+):
+    '''
+    Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
+    storage config dict, create a Borg archive and return Borg's JSON output (if any).
+    '''
+    sources = _expand_directories(
+        location_config['source_directories']
+        + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+    )
+
+    pattern_file = _write_pattern_file(location_config.get('patterns'))
+    exclude_file = _write_pattern_file(
+        _expand_home_directories(location_config.get('exclude_patterns'))
+    )
+    checkpoint_interval = storage_config.get('checkpoint_interval', None)
+    chunker_params = storage_config.get('chunker_params', None)
+    compression = storage_config.get('compression', None)
+    remote_rate_limit = storage_config.get('remote_rate_limit', None)
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+    files_cache = location_config.get('files_cache')
+    default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+    archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
+
+    full_command = (
+        (local_path, 'create')
+        + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
+        + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
+        + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
+        + (('--chunker-params', chunker_params) if chunker_params else ())
+        + (('--compression', compression) if compression else ())
+        + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
+        + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--noatime',) if location_config.get('atime') is False else ())
+        + (('--noctime',) if location_config.get('ctime') is False else ())
+        + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
+        + (('--read-special',) if location_config.get('read_special') else ())
+        + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
+        + (('--files-cache', files_cache) if files_cache else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--list', '--filter', 'AME-') if files and not json and not progress else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
+        + (('--stats',) if stats and not json and not dry_run else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
+        + (('--json',) if json else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (
+            '{repository}::{archive_name_format}'.format(
+                repository=repository, archive_name_format=archive_name_format
+            ),
+        )
+        + sources
+    )
+
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if progress:
+        execute_command_without_capture(full_command, error_on_warnings=False)
+        return
+
+    if json:
+        output_log_level = None
+    elif (stats or files) and logger.getEffectiveLevel() == logging.WARNING:
+        output_log_level = logging.WARNING
+    else:
+        output_log_level = logging.INFO
+
+    return execute_command(full_command, output_log_level, error_on_warnings=False)

borgmatic/borg/environment.py

@@ -0,0 +1,31 @@
+import os
+
+OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'borg_base_directory': 'BORG_BASE_DIR',
+    'borg_config_directory': 'BORG_CONFIG_DIR',
+    'borg_cache_directory': 'BORG_CACHE_DIR',
+    'borg_security_directory': 'BORG_SECURITY_DIR',
+    'borg_keys_directory': 'BORG_KEYS_DIR',
+    'encryption_passcommand': 'BORG_PASSCOMMAND',
+    'encryption_passphrase': 'BORG_PASSPHRASE',
+    'ssh_command': 'BORG_RSH',
+}
+
+DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
+    'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
+}
+
+
+def initialize(storage_config):
+    for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name)
+        if value:
+            os.environ[environment_variable_name] = value
+
+    for (
+        option_name,
+        environment_variable_name,
+    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name, False)
+        os.environ[environment_variable_name] = 'yes' if value else 'no'

borgmatic/borg/extract.py

@@ -0,0 +1,102 @@
+import logging
+import os
+
+from borgmatic.execute import execute_command, execute_command_without_capture
+
+logger = logging.getLogger(__name__)
+
+
+def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg', remote_path=None):
+    '''
+    Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
+    dry-run.
+    '''
+    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
+    verbosity_flags = ()
+    if logger.isEnabledFor(logging.DEBUG):
+        verbosity_flags = ('--debug', '--show-rc')
+    elif logger.isEnabledFor(logging.INFO):
+        verbosity_flags = ('--info',)
+
+    full_list_command = (
+        (local_path, 'list', '--short')
+        + remote_path_flags
+        + lock_wait_flags
+        + verbosity_flags
+        + (repository,)
+    )
+
+    list_output = execute_command(full_list_command, output_log_level=None, error_on_warnings=False)
+
+    try:
+        last_archive_name = list_output.strip().splitlines()[-1]
+    except IndexError:
+        return
+
+    list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
+    full_extract_command = (
+        (local_path, 'extract', '--dry-run')
+        + remote_path_flags
+        + lock_wait_flags
+        + verbosity_flags
+        + list_flag
+        + (
+            '{repository}::{last_archive_name}'.format(
+                repository=repository, last_archive_name=last_archive_name
+            ),
+        )
+    )
+
+    execute_command(full_extract_command, working_directory=None, error_on_warnings=True)
+
+
+def extract_archive(
+    dry_run,
+    repository,
+    archive,
+    paths,
+    location_config,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+    destination_path=None,
+    progress=False,
+    error_on_warnings=True,
+):
+    '''
+    Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
+    restore from the archive, location/storage configuration dicts, optional local and remote Borg
+    paths, and an optional destination path to extract to, extract the archive into the current
+    directory.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'extract')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
+        + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
+        + (tuple(paths) if paths else ())
+    )
+
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if progress:
+        execute_command_without_capture(
+            full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
+        )
+        return
+
+    # Error on warnings by default, as Borg only gives a warning if the restore paths don't exist in
+    # the archive!
+    execute_command(
+        full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
+    )

borgmatic/borg/flags.py

@@ -0,0 +1,31 @@
+import itertools
+
+
+def make_flags(name, value):
+    '''
+    Given a flag name and its value, return it formatted as Borg-compatible flags.
+    '''
+    if not value:
+        return ()
+
+    flag = '--{}'.format(name.replace('_', '-'))
+
+    if value is True:
+        return (flag,)
+
+    return (flag, str(value))
+
+
+def make_flags_from_arguments(arguments, excludes=()):
+    '''
+    Given borgmatic command-line arguments as an instance of argparse.Namespace, and optionally a
+    list of named arguments to exclude, generate and return the corresponding Borg command-line
+    flags as a tuple.
+    '''
+    return tuple(
+        itertools.chain.from_iterable(
+            make_flags(name, value=getattr(arguments, name))
+            for name in sorted(vars(arguments))
+            if name not in excludes and not name.startswith('_')
+        )
+    )

borgmatic/borg/info.py

@@ -0,0 +1,45 @@
+import logging
+
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def display_archives_info(
+    repository, storage_config, info_arguments, local_path='borg', remote_path=None
+):
+    '''
+    Given a local or remote repository path, a storage config dict, and the arguments to the info
+    action, display summary information for Borg archives in the repository or return JSON summary
+    information.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'info')
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not info_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not info_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(info_arguments, excludes=('repository', 'archive'))
+        + (
+            '::'.join((repository, info_arguments.archive))
+            if info_arguments.archive
+            else repository,
+        )
+    )
+
+    return execute_command(
+        full_command,
+        output_log_level=None if info_arguments.json else logging.WARNING,
+        error_on_warnings=False,
+    )

borgmatic/borg/init.py

@@ -0,0 +1,58 @@
+import logging
+import subprocess
+
+from borgmatic.execute import execute_command, execute_command_without_capture
+
+logger = logging.getLogger(__name__)
+
+
+INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
+
+
+def initialize_repository(
+    repository,
+    storage_config,
+    encryption_mode,
+    append_only=None,
+    storage_quota=None,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a storage configuration dict, a Borg encryption mode,
+    whether the repository should be append-only, and the storage quota to use, initialize the
+    repository. If the repository already exists, then log and skip initialization.
+    '''
+    info_command = (
+        (local_path, 'info')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (repository,)
+    )
+    logger.debug(' '.join(info_command))
+
+    try:
+        execute_command(info_command, output_log_level=None)
+        logger.info('Repository already exists. Skipping initialization.')
+        return
+    except subprocess.CalledProcessError as error:
+        if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
+            raise
+
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('init', '')
+
+    init_command = (
+        (local_path, 'init')
+        + (('--encryption', encryption_mode) if encryption_mode else ())
+        + (('--append-only',) if append_only else ())
+        + (('--storage-quota', storage_quota) if storage_quota else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (repository,)
+    )
+
+    # Don't use execute_command() here because it doesn't support interactive prompts.
+    execute_command_without_capture(init_command, error_on_warnings=False)

borgmatic/borg/list.py

@@ -0,0 +1,53 @@
+import logging
+
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+# A hack to convince Borg to exclude archives ending in ".checkpoint". This assumes that a
+# non-checkpoint archive name ends in a digit (e.g. from a timestamp).
+BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[0123456789]'
+
+
+def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
+    '''
+    Given a local or remote repository path, a storage config dict, and the arguments to the list
+    action, display the output of listing Borg archives in the repository or return JSON output. Or,
+    if an archive name is given, listing the files in that archive.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+    if list_arguments.successful:
+        list_arguments.glob_archives = BORG_EXCLUDE_CHECKPOINTS_GLOB
+
+    full_command = (
+        (local_path, 'list')
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(
+            list_arguments, excludes=('repository', 'archive', 'paths', 'successful')
+        )
+        + (
+            '::'.join((repository, list_arguments.archive))
+            if list_arguments.archive
+            else repository,
+        )
+        + (tuple(list_arguments.paths) if list_arguments.paths else ())
+    )
+
+    return execute_command(
+        full_command,
+        output_log_level=None if list_arguments.json else logging.WARNING,
+        error_on_warnings=False,
+    )

borgmatic/borg/mount.py

@@ -0,0 +1,46 @@
+import logging
+
+from borgmatic.execute import execute_command, execute_command_without_capture
+
+logger = logging.getLogger(__name__)
+
+
+def mount_archive(
+    repository,
+    archive,
+    mount_point,
+    paths,
+    foreground,
+    options,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, an optional archive name, a filesystem mount point,
+    zero or more paths to mount from the archive, extra Borg mount options, a storage configuration
+    dict, and optional local and remote Borg paths, mount the archive onto the mount point.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'mount')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--foreground',) if foreground else ())
+        + (('-o', options) if options else ())
+        + (('::'.join((repository, archive)),) if archive else (repository,))
+        + (mount_point,)
+        + (tuple(paths) if paths else ())
+    )
+
+    # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
+    if foreground:
+        execute_command_without_capture(full_command, error_on_warnings=False)
+        return
+
+    execute_command(full_command, error_on_warnings=False)

borgmatic/borg/prune.py

@@ -0,0 +1,75 @@
+import logging
+
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def _make_prune_flags(retention_config):
+    '''
+    Given a retention config dict mapping from option name to value, tranform it into an iterable of
+    command-line name-value flag pairs.
+
+    For example, given a retention config of:
+
+        {'keep_weekly': 4, 'keep_monthly': 6}
+
+    This will be returned as an iterable of:
+
+        (
+            ('--keep-weekly', '4'),
+            ('--keep-monthly', '6'),
+        )
+    '''
+    config = retention_config.copy()
+
+    if 'prefix' not in config:
+        config['prefix'] = '{hostname}-'
+    elif not config['prefix']:
+        config.pop('prefix')
+
+    return (
+        ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
+    )
+
+
+def prune_archives(
+    dry_run,
+    repository,
+    storage_config,
+    retention_config,
+    local_path='borg',
+    remote_path=None,
+    stats=False,
+    files=False,
+):
+    '''
+    Given dry-run flag, a local or remote repository path, a storage config dict, and a
+    retention config dict, prune Borg archives according to the retention policy specified in that
+    configuration.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('prune', '')
+
+    full_command = (
+        (local_path, 'prune')
+        + tuple(element for pair in _make_prune_flags(retention_config) for element in pair)
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--stats',) if stats and not dry_run else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--list',) if files else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (repository,)
+    )
+
+    if (stats or files) and logger.getEffectiveLevel() == logging.WARNING:
+        output_log_level = logging.WARNING
+    else:
+        output_log_level = logging.INFO
+
+    execute_command(full_command, output_log_level=output_log_level, error_on_warnings=False)

borgmatic/borg/umount.py

@@ -0,0 +1,20 @@
+import logging
+
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def unmount_archive(mount_point, local_path='borg'):
+    '''
+    Given a mounted filesystem mount point, and an optional local Borg paths, umount the filesystem
+    from the mount point.
+    '''
+    full_command = (
+        (local_path, 'umount')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (mount_point,)
+    )
+
+    execute_command(full_command, error_on_warnings=True)

borgmatic/commands/arguments.py

@@ -0,0 +1,559 @@
+import collections
+from argparse import ArgumentParser
+
+from borgmatic.config import collect
+
+SUBPARSER_ALIASES = {
+    'init': ['--init', '-I'],
+    'prune': ['--prune', '-p'],
+    'create': ['--create', '-C'],
+    'check': ['--check', '-k'],
+    'extract': ['--extract', '-x'],
+    'mount': ['--mount', '-m'],
+    'umount': ['--umount', '-u'],
+    'restore': ['--restore', '-r'],
+    'list': ['--list', '-l'],
+    'info': ['--info', '-i'],
+}
+
+
+def parse_subparser_arguments(unparsed_arguments, subparsers):
+    '''
+    Given a sequence of arguments, and a subparsers object as returned by
+    argparse.ArgumentParser().add_subparsers(), give each requested action's subparser a shot at
+    parsing all arguments. This allows common arguments like "--repository" to be shared across
+    multiple subparsers.
+
+    Return the result as a dict mapping from subparser name to a parsed namespace of arguments.
+    '''
+    arguments = collections.OrderedDict()
+    remaining_arguments = list(unparsed_arguments)
+    alias_to_subparser_name = {
+        alias: subparser_name
+        for subparser_name, aliases in SUBPARSER_ALIASES.items()
+        for alias in aliases
+    }
+
+    for subparser_name, subparser in subparsers.choices.items():
+        if subparser_name not in remaining_arguments:
+            continue
+
+        canonical_name = alias_to_subparser_name.get(subparser_name, subparser_name)
+
+        # If a parsed value happens to be the same as the name of a subparser, remove it from the
+        # remaining arguments. This prevents, for instance, "check --only extract" from triggering
+        # the "extract" subparser.
+        parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+        for value in vars(parsed).values():
+            if isinstance(value, str):
+                if value in subparsers.choices:
+                    remaining_arguments.remove(value)
+            elif isinstance(value, list):
+                for item in value:
+                    if item in subparsers.choices:
+                        remaining_arguments.remove(item)
+
+        arguments[canonical_name] = parsed
+
+    # If no actions are explicitly requested, assume defaults: prune, create, and check.
+    if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
+        for subparser_name in ('prune', 'create', 'check'):
+            subparser = subparsers.choices[subparser_name]
+            parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+            arguments[subparser_name] = parsed
+
+    return arguments
+
+
+def parse_global_arguments(unparsed_arguments, top_level_parser, subparsers):
+    '''
+    Given a sequence of arguments, a top-level parser (containing subparsers), and a subparsers
+    object as returned by argparse.ArgumentParser().add_subparsers(), parse and return any global
+    arguments as a parsed argparse.Namespace instance.
+    '''
+    # Ask each subparser, one by one, to greedily consume arguments. Any arguments that remain
+    # are global arguments.
+    remaining_arguments = list(unparsed_arguments)
+    present_subparser_names = set()
+
+    for subparser_name, subparser in subparsers.choices.items():
+        if subparser_name not in remaining_arguments:
+            continue
+
+        present_subparser_names.add(subparser_name)
+        unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    # If no actions are explicitly requested, assume defaults: prune, create, and check.
+    if (
+        not present_subparser_names
+        and '--help' not in unparsed_arguments
+        and '-h' not in unparsed_arguments
+    ):
+        for subparser_name in ('prune', 'create', 'check'):
+            subparser = subparsers.choices[subparser_name]
+            unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    # Remove the subparser names themselves.
+    for subparser_name in present_subparser_names:
+        if subparser_name in remaining_arguments:
+            remaining_arguments.remove(subparser_name)
+
+    return top_level_parser.parse_args(remaining_arguments)
+
+
+def parse_arguments(*unparsed_arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance.
+    '''
+    config_paths = collect.get_default_config_paths(expand_home=True)
+    unexpanded_config_paths = collect.get_default_config_paths(expand_home=False)
+
+    global_parser = ArgumentParser(add_help=False)
+    global_group = global_parser.add_argument_group('global arguments')
+
+    global_group.add_argument(
+        '-c',
+        '--config',
+        nargs='*',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(unexpanded_config_paths)
+        ),
+    )
+    global_group.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        help='Deprecated in favor of exclude_patterns within configuration',
+    )
+    global_group.add_argument(
+        '-n',
+        '--dry-run',
+        dest='dry_run',
+        action='store_true',
+        help='Go through the motions, but do not actually write to any repositories',
+    )
+    global_group.add_argument(
+        '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output'
+    )
+    global_group.add_argument(
+        '-v',
+        '--verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=0,
+        help='Display verbose progress to the console (from only errors to very verbose: -1, 0, 1, or 2)',
+    )
+    global_group.add_argument(
+        '--syslog-verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=0,
+        help='Log verbose progress to syslog (from only errors to very verbose: -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given',
+    )
+    global_group.add_argument(
+        '--log-file-verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=0,
+        help='Log verbose progress to log file (from only errors to very verbose: -1, 0, 1, or 2). Only used when --log-file is given',
+    )
+    global_group.add_argument(
+        '--monitoring-verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=1,
+        help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)',
+    )
+    global_group.add_argument(
+        '--log-file',
+        type=str,
+        default=None,
+        help='Write log messages to this file instead of syslog',
+    )
+    global_group.add_argument(
+        '--override',
+        metavar='SECTION.OPTION=VALUE',
+        nargs='+',
+        dest='overrides',
+        help='One or more configuration file options to override with specified values',
+    )
+    global_group.add_argument(
+        '--version',
+        dest='version',
+        default=False,
+        action='store_true',
+        help='Display installed version number of borgmatic and exit',
+    )
+
+    top_level_parser = ArgumentParser(
+        description='''
+            Simple, configuration-driven backup software for servers and workstations. If none of
+            the action options are given, then borgmatic defaults to: prune, create, and check
+            archives.
+            ''',
+        parents=[global_parser],
+    )
+
+    subparsers = top_level_parser.add_subparsers(
+        title='actions',
+        metavar='',
+        help='Specify zero or more actions. Defaults to prune, create, and check. Use --help with action for details:',
+    )
+    init_parser = subparsers.add_parser(
+        'init',
+        aliases=SUBPARSER_ALIASES['init'],
+        help='Initialize an empty Borg repository',
+        description='Initialize an empty Borg repository',
+        add_help=False,
+    )
+    init_group = init_parser.add_argument_group('init arguments')
+    init_group.add_argument(
+        '-e',
+        '--encryption',
+        dest='encryption_mode',
+        help='Borg repository encryption mode',
+        required=True,
+    )
+    init_group.add_argument(
+        '--append-only',
+        dest='append_only',
+        action='store_true',
+        help='Create an append-only repository',
+    )
+    init_group.add_argument(
+        '--storage-quota',
+        dest='storage_quota',
+        help='Create a repository with a fixed storage quota',
+    )
+    init_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    prune_parser = subparsers.add_parser(
+        'prune',
+        aliases=SUBPARSER_ALIASES['prune'],
+        help='Prune archives according to the retention policy',
+        description='Prune archives according to the retention policy',
+        add_help=False,
+    )
+    prune_group = prune_parser.add_argument_group('prune arguments')
+    prune_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    prune_group.add_argument(
+        '--files', dest='files', default=False, action='store_true', help='Show per-file details'
+    )
+    prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    create_parser = subparsers.add_parser(
+        'create',
+        aliases=SUBPARSER_ALIASES['create'],
+        help='Create archives (actually perform backups)',
+        description='Create archives (actually perform backups)',
+        add_help=False,
+    )
+    create_group = create_parser.add_argument_group('create arguments')
+    create_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is backed up',
+    )
+    create_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    create_group.add_argument(
+        '--files', dest='files', default=False, action='store_true', help='Show per-file details'
+    )
+    create_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    check_parser = subparsers.add_parser(
+        'check',
+        aliases=SUBPARSER_ALIASES['check'],
+        help='Check archives for consistency',
+        description='Check archives for consistency',
+        add_help=False,
+    )
+    check_group = check_parser.add_argument_group('check arguments')
+    check_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is checked',
+    )
+    check_group.add_argument(
+        '--repair',
+        dest='repair',
+        default=False,
+        action='store_true',
+        help='Attempt to repair any inconsistencies found (experimental and only for interactive use)',
+    )
+    check_group.add_argument(
+        '--only',
+        metavar='CHECK',
+        choices=('repository', 'archives', 'data', 'extract'),
+        dest='only',
+        action='append',
+        help='Run a particular consistency check (repository, archives, data, or extract) instead of configured checks; can specify flag multiple times',
+    )
+    check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    extract_parser = subparsers.add_parser(
+        'extract',
+        aliases=SUBPARSER_ALIASES['extract'],
+        help='Extract files from a named archive to the current directory',
+        description='Extract a named archive to the current directory',
+        add_help=False,
+    )
+    extract_group = extract_parser.add_argument_group('extract arguments')
+    extract_group.add_argument(
+        '--repository',
+        help='Path of repository to extract, defaults to the configured repository if there is only one',
+    )
+    extract_group.add_argument('--archive', help='Name of archive to extract', required=True)
+    extract_group.add_argument(
+        '--path',
+        '--restore-path',
+        metavar='PATH',
+        nargs='+',
+        dest='paths',
+        help='Paths to extract from archive, defaults to the entire archive',
+    )
+    extract_group.add_argument(
+        '--destination',
+        metavar='PATH',
+        dest='destination',
+        help='Directory to extract files into, defaults to the current directory',
+    )
+    extract_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is extracted',
+    )
+    extract_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    mount_parser = subparsers.add_parser(
+        'mount',
+        aliases=SUBPARSER_ALIASES['mount'],
+        help='Mount files from a named archive as a FUSE filesystem',
+        description='Mount a named archive as a FUSE filesystem',
+        add_help=False,
+    )
+    mount_group = mount_parser.add_argument_group('mount arguments')
+    mount_group.add_argument(
+        '--repository',
+        help='Path of repository to use, defaults to the configured repository if there is only one',
+    )
+    mount_group.add_argument('--archive', help='Name of archive to mount')
+    mount_group.add_argument(
+        '--mount-point',
+        metavar='PATH',
+        dest='mount_point',
+        help='Path where filesystem is to be mounted',
+        required=True,
+    )
+    mount_group.add_argument(
+        '--path',
+        metavar='PATH',
+        nargs='+',
+        dest='paths',
+        help='Paths to mount from archive, defaults to the entire archive',
+    )
+    mount_group.add_argument(
+        '--foreground',
+        dest='foreground',
+        default=False,
+        action='store_true',
+        help='Stay in foreground until ctrl-C is pressed',
+    )
+    mount_group.add_argument('--options', dest='options', help='Extra Borg mount options')
+    mount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    umount_parser = subparsers.add_parser(
+        'umount',
+        aliases=SUBPARSER_ALIASES['umount'],
+        help='Unmount a FUSE filesystem that was mounted with "borgmatic mount"',
+        description='Unmount a mounted FUSE filesystem',
+        add_help=False,
+    )
+    umount_group = umount_parser.add_argument_group('umount arguments')
+    umount_group.add_argument(
+        '--mount-point',
+        metavar='PATH',
+        dest='mount_point',
+        help='Path of filesystem to unmount',
+        required=True,
+    )
+    umount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    restore_parser = subparsers.add_parser(
+        'restore',
+        aliases=SUBPARSER_ALIASES['restore'],
+        help='Restore database dumps from a named archive',
+        description='Restore database dumps from a named archive. (To extract files instead, use "borgmatic extract".)',
+        add_help=False,
+    )
+    restore_group = restore_parser.add_argument_group('restore arguments')
+    restore_group.add_argument(
+        '--repository',
+        help='Path of repository to restore from, defaults to the configured repository if there is only one',
+    )
+    restore_group.add_argument('--archive', help='Name of archive to restore from', required=True)
+    restore_group.add_argument(
+        '--database',
+        metavar='NAME',
+        nargs='+',
+        dest='databases',
+        help='Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each database dump file as it is extracted from archive',
+    )
+    restore_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    list_parser = subparsers.add_parser(
+        'list',
+        aliases=SUBPARSER_ALIASES['list'],
+        help='List archives',
+        description='List archives or the contents of an archive',
+        add_help=False,
+    )
+    list_group = list_parser.add_argument_group('list arguments')
+    list_group.add_argument(
+        '--repository',
+        help='Path of repository to list, defaults to the configured repository if there is only one',
+    )
+    list_group.add_argument('--archive', help='Name of archive to list')
+    list_group.add_argument(
+        '--path',
+        metavar='PATH',
+        nargs='+',
+        dest='paths',
+        help='Paths to list from archive, defaults to the entire archive',
+    )
+    list_group.add_argument(
+        '--short', default=False, action='store_true', help='Output only archive or path names'
+    )
+    list_group.add_argument('--format', help='Format for file listing')
+    list_group.add_argument(
+        '--json', default=False, action='store_true', help='Output results as JSON'
+    )
+    list_group.add_argument(
+        '-P', '--prefix', help='Only list archive names starting with this prefix'
+    )
+    list_group.add_argument(
+        '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob'
+    )
+    list_group.add_argument(
+        '--successful',
+        default=False,
+        action='store_true',
+        help='Only list archive names of successful (non-checkpoint) backups',
+    )
+    list_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    list_group.add_argument(
+        '--first', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '--last', metavar='N', help='List last N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
+    )
+    list_group.add_argument(
+        '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
+    )
+    list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
+    list_group.add_argument(
+        '--patterns-from',
+        metavar='FILENAME',
+        help='Include or exclude paths matching patterns from pattern file, one per line',
+    )
+    list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    info_parser = subparsers.add_parser(
+        'info',
+        aliases=SUBPARSER_ALIASES['info'],
+        help='Display summary information on archives',
+        description='Display summary information on archives',
+        add_help=False,
+    )
+    info_group = info_parser.add_argument_group('info arguments')
+    info_group.add_argument(
+        '--repository',
+        help='Path of repository to show info for, defaults to the configured repository if there is only one',
+    )
+    info_group.add_argument('--archive', help='Name of archive to show info for')
+    info_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    info_group.add_argument(
+        '-P', '--prefix', help='Only show info for archive names starting with this prefix'
+    )
+    info_group.add_argument(
+        '-a',
+        '--glob-archives',
+        metavar='GLOB',
+        help='Only show info for archive names matching this glob',
+    )
+    info_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    info_group.add_argument(
+        '--first',
+        metavar='N',
+        help='Show info for first N archives after other filters are applied',
+    )
+    info_group.add_argument(
+        '--last', metavar='N', help='Show info for first N archives after other filters are applied'
+    )
+    info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    arguments = parse_subparser_arguments(unparsed_arguments, subparsers)
+    arguments['global'] = parse_global_arguments(unparsed_arguments, top_level_parser, subparsers)
+
+    if arguments['global'].excludes_filename:
+        raise ValueError(
+            'The --excludes option has been replaced with exclude_patterns in configuration'
+        )
+
+    if 'init' in arguments and arguments['global'].dry_run:
+        raise ValueError('The init action cannot be used with the --dry-run option')
+
+    if 'list' in arguments and arguments['list'].glob_archives and arguments['list'].successful:
+        raise ValueError('The --glob-archives and --successful options cannot be used together')
+
+    if (
+        'list' in arguments
+        and 'info' in arguments
+        and arguments['list'].json
+        and arguments['info'].json
+    ):
+        raise ValueError('With the --json option, list and info actions cannot be used together')
+
+    return arguments

borgmatic/commands/borgmatic.py

@@ -1,73 +1,677 @@
-from __future__ import print_function
-from argparse import ArgumentParser
+import collections
+import json
+import logging
 import os
-from subprocess import CalledProcessError
 import sys
+from subprocess import CalledProcessError
 
-from borgmatic import borg
-from borgmatic.config import collect, convert, validate
+import colorama
+import pkg_resources
 
+from borgmatic.borg import check as borg_check
+from borgmatic.borg import create as borg_create
+from borgmatic.borg import environment as borg_environment
+from borgmatic.borg import extract as borg_extract
+from borgmatic.borg import info as borg_info
+from borgmatic.borg import init as borg_init
+from borgmatic.borg import list as borg_list
+from borgmatic.borg import mount as borg_mount
+from borgmatic.borg import prune as borg_prune
+from borgmatic.borg import umount as borg_umount
+from borgmatic.commands.arguments import parse_arguments
+from borgmatic.config import checks, collect, convert, validate
+from borgmatic.hooks import command, dispatch, dump, monitor
+from borgmatic.logger import configure_logging, should_do_markup
+from borgmatic.signals import configure_signals
+from borgmatic.verbosity import verbosity_to_log_level
+
+logger = logging.getLogger(__name__)
 
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
-DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d']
-DEFAULT_EXCLUDES_PATH = '/etc/borgmatic/excludes'
 
 
-def parse_arguments(*arguments):
+def run_configuration(config_filename, config, arguments):
     '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    parser = ArgumentParser()
-    parser.add_argument(
-        '-c', '--config',
-        nargs='+',
-        dest='config_paths',
-        default=DEFAULT_CONFIG_PATHS,
-        help='Configuration filenames or directories, defaults to: {}'.format(' '.join(DEFAULT_CONFIG_PATHS)),
-    )
-    parser.add_argument(
-        '--excludes',
-        dest='excludes_filename',
-        help='Excludes filename, deprecated in favor of exclude_patterns within configuration',
-    )
-    parser.add_argument(
-        '-v', '--verbosity',
-        type=int,
-        help='Display verbose progress (1 for some, 2 for lots)',
-    )
+    Given a config filename, the corresponding parsed config dict, and command-line arguments as a
+    dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
+    backups, consistency checks, and/or other actions.
 
-    return parser.parse_args(arguments)
+    Yield a combination of:
+
+      * JSON output strings from successfully executing any actions that produce JSON
+      * logging.LogRecord instances containing errors from any actions or backup hooks that fail
+    '''
+    (location, storage, retention, consistency, hooks) = (