Database dump hooks documentation (#225).

https://yaml.org/spec/1.2/spec.html#id2780069

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.tox

.drone.yml

@@ -0,0 +1,57 @@
+---
+kind: pipeline
+name: python-3-5-alpine-3-10
+
+steps:
+- name: build
+  image: python:3.5-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-6-alpine-3-10
+
+steps:
+- name: build
+  image: python:3.6-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-10
+
+steps:
+- name: build
+  image: python:3.7-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-7
+
+steps:
+- name: build
+  image: python:3.7-alpine3.7
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: documentation
+
+steps:
+- name: build
+  image: plugins/docker
+  settings:
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+    repo: witten/borgmatic-docs
+    dockerfile: docs/Dockerfile
+  when:
+    branch:
+      - master

.eleventy.js

@@ -0,0 +1,42 @@
+const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
+const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language");
+
+module.exports = function(eleventyConfig) {
+    eleventyConfig.addPlugin(pluginSyntaxHighlight);
+    eleventyConfig.addPlugin(inclusiveLangPlugin);
+
+    let markdownIt = require("markdown-it");
+    let markdownItAnchor = require("markdown-it-anchor");
+    let markdownItReplaceLink = require("markdown-it-replace-link");
+
+    let markdownItOptions = {
+        html: true,
+        breaks: false,
+        linkify: true,
+        replaceLink: function (link, env) {
+            if (process.env.NODE_ENV == "production") {
+                return link;
+            }
+            return link.replace('https://torsion.org/borgmatic/', 'http://localhost:8080/');
+        }
+    };
+    let markdownItAnchorOptions = {
+        permalink: true,
+        permalinkClass: "direct-link"
+    };
+
+    eleventyConfig.setLibrary(
+        "md",
+        markdownIt(markdownItOptions)
+            .use(markdownItAnchor, markdownItAnchorOptions)
+            .use(markdownItReplaceLink)
+    );
+
+    return {
+        templateFormats: [
+          "md",
+          "txt"
+        ]
+    }
+};
+

.gitea/issue_template.md

@@ -0,0 +1,31 @@
+#### What I'm trying to do and why
+
+#### Steps to reproduce (if a bug)
+
+Include (sanitized) borgmatic configuration files if applicable.
+
+#### Actual behavior (if a bug)
+
+Include (sanitized) `--verbosity 2` output if applicable.
+
+#### Expected behavior (if a bug)
+
+#### Other notes / implementation ideas
+
+#### Environment
+
+**borgmatic version:** [version here]
+
+Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
+
+**borgmatic installation method:** [e.g., Debian package, Docker container, etc.]
+
+**Borg version:** [version here]
+
+Use `sudo borg --version`
+
+**Python version:** [version here]
+
+Use `python3 --version`
+
+**operating system and version:** [OS here]

.gitignore

@@ -3,6 +3,9 @@
 *.swp
 .cache
 .coverage
+.pytest_cache
 .tox
-build
-dist
+__pycache__
+build/
+dist/
+pip-wheel-metadata/

.hgignore

@@ -1,9 +0,0 @@
-syntax: glob
-*.egg-info
-*.pyc
-*.swp
-.cache
-.coverage
-.tox
-build
-dist

.hgtags

@@ -1,40 +0,0 @@
-467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
-7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3
-4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4
-b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
-b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
-aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
-aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
-569aef47a9b25c55b13753f94706f5d330219995 0.0.5
-569aef47a9b25c55b13753f94706f5d330219995 0.0.5
-a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
-7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
-cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
-38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0
-ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
-ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
-7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1
-83067f995dd391e38544a7722dc3b254b59c5521 0.1.2
-acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
-6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
-6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
-e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
-0afff209b902698c2266986129d6dc9f5f913101 0.1.5
-4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6
-5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7
-977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files
-0000000000000000000000000000000000000000 github/yaml_config_files
-28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master
-dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8
-0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0
-de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
-9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2
-32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3
-5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0
-7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1
-f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2
-3f838f661546e04529b453aa443529b432afc243 1.1.3
-3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4
-64ca13bfe050f656b44ed2eb1c3db045bfddd133 1.1.5
-4daa944c122c572b9b56bfcac3f4e2869181c630 1.1.6
-ec7949a14a2051616f7cdcb8e05555f02e024ae8 1.1.7

AUTHORS

@@ -1,8 +1,12 @@
 Dan Helfman <witten@torsion.org>: Main developer
 
 Alexander Görtz: Python 3 compatibility
+Florian Lindner: Logging rewrite
 Henning Schroeder: Copy editing
-Michele Lazzeri: Custom archive names
-Robin `ypid` Schneider: Support additional options of Borg
-Scott Squires: Custom archive names
 Johannes Feichtner: Support for user hooks
+Michele Lazzeri: Custom archive names
+Nick Whyte: Support prefix filtering for archive consistency checks
+newtonne: Read encryption password from external file
+Robin `ypid` Schneider: Support additional options of Borg and add validate-borgmatic-config command
+Scott Squires: Custom archive names
+Thomas LÉVEIL: Support for a keep_minutely prune option. Support for the --json option

MANIFEST.in

@@ -1 +1,2 @@
 include borgmatic/config/schema.yaml
+graft sample/systemd

NEWS

@@ -1,7 +1,314 @@
+1.4.0
+ * #225: Database dump hooks for PostgreSQL, so you can easily dump your databases before backups
+   run.
+ * #230: Rename "borgmatic list --pattern-from" flag to "--patterns-from" to match Borg.
+
+1.3.26
+ * #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
+   (non-checkpoint) archives.
+
+1.3.25
+ * #223: Dead man's switch to detect when backups start failing silently, implemented via
+   healthchecks.io hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
+ * Documentation on monitoring and alerting options for borgmatic backups:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/
+ * Automatically rewrite links when developing on documentation locally.
+
+1.3.24
+ * #86: Add "borgmatic list --successful" flag to only list successful (non-checkpoint) archives.
+ * Add a suggestion form to all documentation pages, so users can submit ideas for improving the
+   documentation.
+ * Update documentation link to community Arch Linux borgmatic package.
+
+1.3.23
+ * #174: More detailed error alerting via runtime context available in "on_error" hook.
+
+1.3.22
+ * #144: When backups to one of several repositories fails, keep backing up to the other
+   repositories and report errors afterwards.
+
+1.3.21
+ * #192: User-defined hooks for global setup or cleanup that run before/after all actions. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+
+1.3.20
+ * #205: More robust sample systemd service: boot delay, network dependency, lowered CPU/IO
+   priority, etc.
+ * #221: Fix "borgmatic create --progress" output so that it updates on the console in real-time.
+
+1.3.19
+ * #219: Fix visibility of "borgmatic prune --stats" output.
+
+1.3.18
+ * #220: Fix regression of argument parsing for default actions.
+
+1.3.17
+ * #217: Fix error with "borgmatic check --only" command-line flag with "extract" consistency check.
+
+1.3.16
+ * #210: Support for Borg check --verify-data flag via borgmatic "data" consistency check.
+ * #210: Override configured consistency checks via "borgmatic check --only" command-line flag.
+ * When generating sample configuration with generate-borgmatic-config, add a space after each "#"
+   comment indicator.
+
+1.3.15
+ * #208: Fix for traceback when the "checks" option has an empty value.
+ * #209: Bypass Borg error about a moved repository via "relocated_repo_access_is_ok" option in
+   borgmatic storage configuration section.
+ * #213: Reorder arguments passed to Borg to fix duplicate directories when using Borg patterns.
+ * #214: Fix for hook erroring with exit code 1 not being interpreted as an error.
+
+1.3.14
+ * #204: Do not treat Borg warnings (exit code 1) as failures.
+ * When validating configuration files, require strings instead of allowing any scalar type.
+
+1.3.13
+ * #199: Add note to documentation about using spaces instead of tabs for indentation, as YAML does
+   not allow tabs.
+ * #203: Fix compatibility with ruamel.yaml 0.16.x.
+ * If a "prefix" option in borgmatic's configuration has an empty value (blank or ""), then disable
+   default prefix.
+
+1.3.12
+ * Only log to syslog when run from a non-interactive console (e.g. a cron job).
+ * Remove unicode byte order mark from syslog output so it doesn't show up as a literal in rsyslog
+   output. See discussion on #197.
+
+1.3.11
+ * #193: Pass through several "borg list" and "borg info" flags like --short, --format, --sort-by,
+   --first, --last, etc. via borgmatic command-line flags.
+ * Add borgmatic info --repository and --archive command-line flags to display info for individual
+   repositories or archives.
+ * Support for Borg --noatime, --noctime, and --nobirthtime flags via corresponding options in
+   borgmatic configuration location section.
+
+1.3.10
+ * #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero.
+
+1.3.9
+ * #195: Switch to command-line actions as more traditional sub-commands, e.g. "borgmatic create",
+   "borgmatic prune", etc. However, the classic dashed options like "--create" still work!
+
+1.3.8
+ * #191: Disable console color via "color" option in borgmatic configuration output section.
+
+1.3.7
+ * #196: Fix for unclear error message for invalid YAML merge include.
+ * #197: Don't color syslog output.
+ * Change default syslog verbosity to show errors only.
+
+1.3.6
+ * #53: Log to syslog in addition to existing console logging. Add --syslog-verbosity flag to
+   customize the log level. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/
+ * #178: Look for .yml configuration file extension in addition to .yaml.
+ * #189: Set umask used when executing hooks via "umask" option in borgmatic hooks section.
+ * Remove Python cache files before each Tox run.
+ * Add #borgmatic Freenode IRC channel to documentation.
+ * Add Borg/borgmatic hosting providers section to documentation.
+ * Add files for building documentation into a Docker image for web serving.
+ * Upgrade project build server from Drone 0.8 to 1.1.
+ * Build borgmatic documentation during continuous integration.
+ * We're nearly at 500 ★s on GitHub. We can do this!
+
+1.3.5
+ * #153: Support for various Borg directory environment variables (BORG_CONFIG_DIR, BORG_CACHE_DIR,
+   etc.) via options in borgmatic's storage configuration.
+ * #177: Fix for regression with missing verbose log entries.
+
+1.3.4
+ * Part of #125: Color borgmatic (but not Borg) output when using an interactive terminal.
+ * #166: Run tests for all installed versions of Python.
+ * #168: Update README with continuous integration badge.
+ * #169: Automatically sort Python imports in code.
+ * Document installing borgmatic with pip install --user instead of a system Python install.
+ * Get more reproducible builds by pinning the versions of pip and tox used to run tests.
+ * Factor out build/test configuration from tox.ini file.
+
+1.3.3
+ * Add validate-borgmatic-config command, useful for validating borgmatic config generated by
+   configuration management or even edited by hand.
+
+1.3.2
+ * #160: Fix for hooks executing when using --dry-run. Now hooks are skipped during a dry run.
+
+1.3.1
+ * #155: Fix for invalid JSON output when using multiple borgmatic configuration files.
+ * #157: Fix for seemingly random filename ordering when running through a directory of
+   configuration files.
+ * Fix for empty JSON output when using --create --json.
+ * Now capturing Borg output only when --json flag is used. Previously, borgmatic delayed Borg
+   output even without the --json flag.
+
+1.3.0
+ * #148: Configuration file includes and merging via "!include" tag to support reuse of common
+   options across configuration files.
+
+1.2.18
+ * #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
+   borgmatic's location section.
+
+1.2.17
+ * #140: List the files within an archive via --list --archive option.
+
+1.2.16
+ * #119: Include a sample borgmatic configuration file in the documentation.
+ * #123: Support for Borg archive restoration via borgmatic --extract command-line flag.
+ * Refactor documentation into multiple separate pages for clarity and findability.
+ * Organize options within command-line help into logical groups.
+ * Exclude tests from distribution packages.
+
+1.2.15
+ * #127: Remove date echo from schema example, as it's not a substitute for real logging.
+ * #132: Leave exclude_patterns glob expansion to Borg, since doing it in borgmatic leads to
+   confusing behavior.
+ * #136: Handle and format validation errors raised during argument parsing.
+ * #138: Allow use of --stats flag when --create or --prune flags are implied.
+
+1.2.14
+ * #103: When generating sample configuration with generate-borgmatic-config, document the defaults
+   for each option.
+ * #116: When running multiple configuration files, attempt all configuration files even if one of
+   them errors. Log a summary of results at the end.
+ * Add borgmatic --version command-line flag to get the current installed version number.
+
+1.2.13
+ * #100: Support for --stats command-line flag independent of --verbosity.
+ * #117: With borgmatic --init command-line flag, proceed without erroring if a repository already
+   exists.
+
+1.2.12
+ * #110: Support for Borg repository initialization via borgmatic --init command-line flag.
+ * #111: Update Borg create --filter values so a dry run lists files to back up.
+ * #113: Update README with link to a new/forked Docker image.
+ * Prevent deprecated --excludes command-line option from being used.
+ * Refactor README a bit to flow better for first-time users.
+ * Update README with a few additional borgmatic packages (Debian and Ubuntu).
+
+1.2.11
+ * #108: Support for Borg create --progress via borgmatic command-line flag.
+
+1.2.10
+ * #105: Support for Borg --chunker-params create option via "chunker_params" option in borgmatic's
+   storage section.
+
+1.2.9
+ * #102: Fix for syntax error that occurred in Python 3.5 and below.
+ * Make automated tests support running in Python 3.5.
+
+1.2.8
+ * #73: Enable consistency checks for only certain repositories via "check_repositories" option in
+   borgmatic's consistency configuration. Handy for large repositories that take forever to check.
+ * Include link to issue tracker within various command output.
+ * Run continuous integration tests on a matrix of Python and Borg versions.
+
+1.2.7
+ * #98: Support for Borg --keep-secondly prune option.
+ * Use Black code formatter and Flake8 code checker as part of running automated tests.
+ * Add an end-to-end automated test that actually integrates with Borg.
+ * Set up continuous integration for borgmatic automated tests on projects.evoworx.org.
+
+1.2.6
+ * Fix generated configuration to also include a "keep_daily" value so pruning works out of the
+   box.
+
+1.2.5
+ * #57: When generating sample configuration with generate-borgmatic-config, comment out all
+   optional configuration so as to streamline the initial configuration process.
+
+1.2.4
+ * Fix for archive checking traceback due to parameter mismatch.
+
+1.2.3
+ * #64, #90, #92: Rewrite of logging system. Now verbosity flags passed to Borg are derived from
+   borgmatic's log level. Note that the output of borgmatic might slightly change.
+ * Part of #80: Support for Borg create --read-special via "read_special" option in borgmatic's
+   location configuration.
+ * #87: Support for Borg create --checkpoint-interval via "checkpoint_interval" option in
+   borgmatic's storage configuration.
+ * #88: Fix declared pykwalify compatibility version range in setup.py to prevent use of ancient
+   versions of pykwalify with large version numbers.
+ * #89: Pass --show-rc option to Borg when at highest verbosity level.
+ * #94: Support for Borg --json option via borgmatic command-line to --create archives.
+
+1.2.2
+ * #85: Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52, which manifested in
+   borgmatic as a pykwalify RuleError.
+
+1.2.1
+ * Skip before/after backup hooks when only doing --prune, --check, --list, and/or --info.
+ * #71: Support for XDG_CONFIG_HOME environment variable for specifying alternate user ~/.config/
+   path.
+ * #74, #83: Support for Borg --json option via borgmatic command-line to --list archives or show
+   archive --info in JSON format, ideal for programmatic consumption.
+ * #38, #76: Upgrade ruamel.yaml compatibility version range and fix support for Python 3.7.
+ * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files,
+   editor swap files, etc.
+ * #81: Document user-defined hooks run before/after backup, or on error.
+ * Add code style guidelines to the documention.
+
+1.2.0
+ * #61: Support for Borg --list option via borgmatic command-line to list all archives.
+ * #61: Support for Borg --info option via borgmatic command-line to display summary information.
+ * #62: Update README to mention other ways of installing borgmatic.
+ * Support for Borg --prefix option for consistency checks via "prefix" option in borgmatic's
+   consistency configuration.
+ * Add introductory screencast link to documentation.
+ * #59: Ignore "check_last" and consistency "prefix" when "archives" not in consistency checks.
+ * #60: Add "Persistent" flag to systemd timer example.
+ * #63: Support for Borg --nobsdflags option to skip recording bsdflags (e.g. NODUMP, IMMUTABLE) in
+   archive.
+ * #69: Support for Borg prune --umask option using value of existing "umask" option in borgmatic's
+   storage configuration.
+ * Update tox.ini to only assume Python 3.x instead of Python 3.4 specifically.
+ * Add ~/.config/borgmatic/config.yaml to default configuration path probing.
+ * Document how to develop on and contribute to borgmatic.
+
+1.1.15
+ * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
+ * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
+   Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298
+ * #55: Fix for missing tags/releases on Gitea and GitHub project hosting.
+ * #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock.
+ * #58: Support for using tilde in exclude_patterns to reference home directory.
+
+1.1.14
+ * #49: Fix for typo in --patterns-from option.
+ * #47: Support for Borg --dry-run option via borgmatic command-line.
+
+1.1.13
+ * #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository",
+   "archives", and "extract") are specified in borgmatic configuration.
+ * #48: Add "local_path" to configuration for specifying an alternative Borg executable path.
+ * #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed
+   includes/excludes.
+ * Moved issue tracker from Taiga to integrated Gitea tracker at
+   https://projects.torsion.org/witten/borgmatic/issues
+
+1.1.12
+ * #46: Declare dependency on pykwalify 1.6 or above, as older versions yield "Unknown key: version"
+   rule errors.
+ * Support for Borg --keep-minutely prune option.
+
+1.1.11
+ * #26: Add "ssh_command" to configuration for specifying a custom SSH command or options.
+ * Fix for incorrect /etc/borgmatic.d/ configuration path probing on macOS. This problem manifested
+   as an error on startup: "[Errno 2] No such file or directory: '/etc/borgmatic.d'".
+
+1.1.10
+ * Pass several Unix signals through to child processes like Borg. This means that Borg now properly
+   shuts down if borgmatic is terminated (e.g. due to a system suspend).
+ * #30: Support for using tilde in repository paths to reference home directory.
+ * #43: Support for Borg --files-cache option for setting the files cache operation mode.
+ * #45: Support for Borg --remote-ratelimit option for limiting upload rate.
+ * Log invoked Borg commands when at highest verbosity level.
+
 1.1.9
- * #16, #38: Support for user-defined hooks before/after backup, or on error.
- * #33: Improve clarity of logging spew at high verbosity levels.
- * #29: Support for using tilde in source directory path to reference home directory.
+ * #17, #39: Support for user-defined hooks before/after backup, or on error.
+ * #34: Improve clarity of logging spew at high verbosity levels.
+ * #30: Support for using tilde in source directory path to reference home directory.
  * Require "prefix" in retention section when "archive_name_format" is set. This is to avoid
    accidental pruning of archives with a different archive name format. For similar reasons, default
    "prefix" to "{hostname}-" if not specified.
@@ -9,43 +316,43 @@
  * Update dead links to Borg documentation.
 
 1.1.8
- * #39: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default
+ * #40: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default
    config paths.
 
 1.1.7
 
- * #28: Add "archive_name_format" to configuration for customizing archive names.
+ * #29: Add "archive_name_format" to configuration for customizing archive names.
  * Fix for traceback when "exclude_from" value is empty in configuration file.
  * When pruning, make highest verbosity level list archives kept and pruned.
  * Clarification of Python 3 pip usage in documentation.
 
 1.1.6
 
- * #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.
+ * #13, #36: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.
 
 1.1.5
 
- * #34: New "extract" consistency check that performs a dry-run extraction of the most recent
+ * #35: New "extract" consistency check that performs a dry-run extraction of the most recent
    archive.
 
 1.1.4
 
- * #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or
+ * #18: Added command-line flags for performing a borgmatic run with only pruning, creating, or
    checking enabled. This supports use cases like running consistency checks from a different cron
    job with a different frequency, or running pruning with a different verbosity level.
 
 1.1.3
 
- * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
+ * #15: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
  * Fix for generate-borgmatic-config writing config with invalid one_file_system value.
 
 1.1.2
 
- * #32: Fix for passing check_last as integer to subprocess when calling Borg.
+ * #33: Fix for passing check_last as integer to subprocess when calling Borg.
 
 1.1.1
 
- * Part of #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of
+ * Part of #33: Fix for upgrade-borgmatic-config converting check_last option as a string instead of
    an integer.
  * Fix for upgrade-borgmatic-config erroring when consistency checks option is not present.
 
@@ -54,8 +361,8 @@
  * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade.
  * Added generate-borgmatic-config command for initial config creation.
  * Dropped Python 2 support. Now Python 3 only.
- * #18: Fix for README mention of sample files not included in package.
- * #22: Sample files for triggering borgmatic from a systemd timer.
+ * #19: Fix for README mention of sample files not included in package.
+ * #23: Sample files for triggering borgmatic from a systemd timer.
  * Support for backing up to multiple repositories.
  * To free up space, now pruning backups prior to creating a new backup.
  * Enabled test coverage output during tox runs.
@@ -63,15 +370,15 @@
 
 1.0.3
 
- * #21: Fix for verbosity flag not actually causing verbose output.
+ * #22: Fix for verbosity flag not actually causing verbose output.
 
 1.0.2
 
- * #20: Fix for traceback when remote_path option is missing.
+ * #21: Fix for traceback when remote_path option is missing.
 
 1.0.1
 
- * #19: Support for Borg's --remote-path option to use an alternate Borg
+ * #20: Support for Borg's --remote-path option to use an alternate Borg
    executable. See sample/config.
 
 1.0.0
@@ -93,13 +400,13 @@
 
 0.1.7
 
- * #11: Fixed parsing of punctuation in configuration file.
+ * #12: Fixed parsing of punctuation in configuration file.
  * Better error message when configuration file is missing.
 
 0.1.6
 
- * #9: New configuration option for the encryption passphrase.
- * #10: Support for Borg's new archive compression feature.
+ * #10: New configuration option for the encryption passphrase.
+ * #11: Support for Borg's new archive compression feature.
 
 0.1.5
 
@@ -111,7 +418,7 @@
 
 0.1.3
 
- * #1: Add support for "borg check --last N" to Borg backend.
+ * #2: Add support for "borg check --last N" to Borg backend.
 
 0.1.2
 

README.md

@@ -1,13 +1,20 @@
-<img src="https://projects.torsion.org/witten/borgmatic/raw/master/static/borgmatic.png" width="150px" style="float: right; padding-left: 1em;">
+---
+title: borgmatic
+permalink: index.html
+---
+<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master)</a>
 
 ## Overview
 
-borgmatic (formerly atticmatic) is a simple Python wrapper script for the
-[Borg](https://borgbackup.readthedocs.org/en/stable/) backup software that
-initiates a backup, prunes any old backups according to a retention policy,
-and validates backups for consistency. The script supports specifying your
-settings in a declarative configuration file rather than having to put them
-all on the command-line, and handles common errors.
+<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
+
+borgmatic is simple, configuration-driven backup software for servers and
+workstations. Backup all of your machines from the command-line or scheduled
+jobs. No GUI required. Built atop [Borg Backup](https://www.borgbackup.org/),
+borgmatic initiates a backup, prunes any old backups according to a retention
+policy, and validates backups for consistency. borgmatic supports specifying
+your settings in a declarative configuration file, rather than having to put
+them all on the command-line, and handles common errors.
 
 Here's an example config file:
 
@@ -34,308 +41,88 @@ retention:
     keep_monthly: 6
 
 consistency:
-    # List of consistency checks to run: "repository", "archives", or both.
+    # List of consistency checks to run: "repository", "archives", etc.
     checks:
         - repository
         - archives
+
+hooks:
+    # Preparation scripts to run, databases to dump, and monitoring to perform.
+    before_backup:
+        - prepare-for-backup.sh
+    postgresql_databases:
+        - name: users
+    healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
 ```
 
 borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
 available](https://projects.torsion.org/witten/borgmatic). It's also mirrored
 on [GitHub](https://github.com/witten/borgmatic) for convenience.
 
+Want to see borgmatic in action? Check out the <a
+href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
 
-## Installation
+<script src="https://asciinema.org/a/203761.js" id="asciicast-203761" async></script>
 
-To get up and running, follow the [Borg Quick
-Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
-a repository on a local or remote host. Note that if you plan to run
-borgmatic on a schedule with cron, and you encrypt your Borg repository with
-a passphrase instead of a key file, you'll need to set the borgmatic
-`encryption_passphrase` configuration variable. See the repository encryption
-section of the Quick Start for more info.
 
-If the repository is on a remote host, make sure that your local root user has
-key-based ssh access to the desired user account on the remote host.
+## How-to guides
 
-To install borgmatic, run the following command to download and install it:
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/) ⬅ *Start here!*
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+ * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+ * [Restore a backup](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/)
+ * [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
 
-```bash
-sudo pip3 install --upgrade borgmatic
-```
 
-Note that your pip binary may have a different name than "pip3". Make sure
-you're using Python 3, as borgmatic does not support Python 2.
+## Reference guides
 
-### Docker
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)
 
-If you would like to run borgmatic within Docker, please take a look at
-[b3vis/borgmatic](https://hub.docker.com/r/b3vis/borgmatic/) for more
-information.
 
-## Configuration
+## Hosting providers
 
-After you install borgmatic, generate a sample configuration file:
+Need somewhere to store your encrypted offsite backups? The following hosting
+providers include specific support for Borg/borgmatic. Using these links and
+services helps support borgmatic development and hosting. (These are referral
+links, but without any tracking scripts or cookies.)
 
-```bash
-sudo generate-borgmatic-config
-```
+<ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+</ul>
 
-This generates a sample configuration file at /etc/borgmatic/config.yaml (by
-default). You should edit the file to suit your needs, as the values are just
-representative. All fields are optional except where indicated, so feel free
-to remove anything you don't need.
+## Support and contributing
 
-You can also have a look at the [full configuration
-schema](https://projects.torsion.org/witten/borgmatic/src/master/borgmatic/config/schema.yaml)
-for the authoritative set of all configuration options. This is handy if
-borgmatic has added new options since you originally created your
-configuration file.
+### Issues
 
+You've got issues? Or an idea for a feature enhancement? We've got an [issue
+tracker](https://projects.torsion.org/witten/borgmatic/issues). In order to
+create a new issue or comment on an issue, you'll need to [login
+first](https://projects.torsion.org/user/login). Note that you can login with
+an existing GitHub account if you prefer.
 
-### Multiple configuration files
-
-A more advanced usage is to create multiple separate configuration files and
-place each one in an /etc/borgmatic.d directory. For instance:
-
-```bash
-sudo mkdir /etc/borgmatic.d
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
-```
-
-With this approach, you can have entirely different backup policies for
-different applications on your system. For instance, you may want one backup
-configuration for your database data directory, and a different configuration
-for your user home directories.
-
-When you set up multiple configuration files like this, borgmatic will run
-each one in turn from a single borgmatic invocation. This includes, by
-default, the traditional /etc/borgmatic/config.yaml as well.
-
-
-## Upgrading
-
-In general, all you should need to do to upgrade borgmatic is run the
-following:
-
-```bash
-sudo pip3 install --upgrade borgmatic
-```
-
-However, see below about special cases.
-
-
-### Upgrading from borgmatic 1.0.x
-
-borgmatic changed its configuration file format in version 1.1.0 from
-INI-style to YAML. This better supports validation, and has a more natural way
-to express lists of values. To upgrade your existing configuration, first
-upgrade to the new version of borgmatic.
-
-As of version 1.1.0, borgmatic no longer supports Python 2. If you were
-already running borgmatic with Python 3, then you can simply upgrade borgmatic
-in-place:
-
-```bash
-sudo pip3 install --upgrade borgmatic
-```
-
-But if you were running borgmatic with Python 2, uninstall and reinstall instead:
-
-```bash
-sudo pip uninstall borgmatic
-sudo pip3 install borgmatic
-```
-
-The pip binary names for different versions of Python can differ, so the above
-commands may need some tweaking to work on your machine.
-
-
-Once borgmatic is upgraded, run:
-
-```bash
-sudo upgrade-borgmatic-config
-```
-
-That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
-(by default) using the values from both your existing configuration and
-excludes files. The new version of borgmatic will consume the YAML
-configuration file instead of the old one.
-
-
-### Upgrading from atticmatic
-
-You can ignore this section if you're not an atticmatic user (the former name
-of borgmatic).
-
-borgmatic only supports Borg now and no longer supports Attic. So if you're
-an Attic user, consider switching to Borg. See the [Borg upgrade
-command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
-for more information. Then, follow the instructions above about setting up
-your borgmatic configuration files.
-
-If you were already using Borg with atticmatic, then you can easily upgrade
-from atticmatic to borgmatic. Simply run the following commands:
-
-```bash
-sudo pip3 uninstall atticmatic
-sudo pip3 install borgmatic
-```
-
-That's it! borgmatic will continue using your /etc/borgmatic configuration
-files.
-
-
-## Usage
-
-You can run borgmatic and start a backup simply by invoking it without
-arguments:
-
-```bash
-borgmatic
-```
-
-This will also prune any old backups as per the configured retention policy,
-and check backups for consistency problems due to things like file damage.
-
-If you'd like to see the available command-line arguments, view the help:
-
-```bash
-borgmatic --help
-```
-
-Note that borgmatic prunes archives *before* creating an archive, so as to
-free up space for archiving. This means that when a borgmatic run finishes,
-there may still be prune-able archives. Not to worry, as they will get cleaned
-up at the start of the next run.
-
-### Verbosity
-
-By default, the backup will proceed silently except in the case of errors. But
-if you'd like to to get additional information about the progress of the
-backup as it proceeds, use the verbosity option:
-
-```bash
-borgmatic --verbosity 1
-```
-
-Or, for even more progress spew:
-
-```bash
-borgmatic --verbosity 2
-```
-
-### À la carte
-
-If you want to run borgmatic with only pruning, creating, or checking enabled,
-the following optional flags are available:
-
-```bash
-borgmatic --prune
-borgmatic --create
-borgmatic --check
-```
-
-You can run with only one of these flags provided, or you can mix and match
-any number of them. This supports use cases like running consistency checks
-from a different cron job with a different frequency, or running pruning with
-a different verbosity level.
-
-
-## Autopilot
-
-If you want to run borgmatic automatically, say once a day, the you can
-configure a job runner to invoke it periodically.
-
-### cron
-
-If you're using cron, download the [sample cron
-file](https://projects.torsion.org/witten/borgmatic/raw/master/sample/cron/borgmatic).
-Then, from the directory where you downloaded it:
-
-```bash
-sudo mv borgmatic /etc/cron.d/borgmatic
-sudo chmod +x /etc/cron.d/borgmatic
-```
-
-You can modify the cron file if you'd like to run borgmatic more or less frequently.
-
-### systemd
-
-If you're using systemd instead of cron to run jobs, download the [sample
-systemd service
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.service)
-and the [sample systemd timer
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.timer).
-Then, from the directory where you downloaded them:
-
-```bash
-sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
-sudo systemctl enable borgmatic.timer
-sudo systemctl start borgmatic.timer
-```
-
-Feel free to modify the timer file based on how frequently you'd like
-borgmatic to run.
-
-
-## Running tests
-
-First install tox, which is used for setting up testing environments:
-
-```bash
-pip3 install tox
-```
-
-Then, to actually run tests, run:
-
-```bash
-tox
-```
-
-
-## Troubleshooting
-
-### Broken pipe with remote repository
-
-When running borgmatic on a large remote repository, you may receive errors
-like the following, particularly while "borg check" is validating backups for
-consistency:
-
-```text
-    Write failed: Broken pipe
-    borg: Error: Connection closed by remote host
-```
-
-This error can be caused by an ssh timeout, which you can rectify by adding
-the following to the ~/.ssh/config file on the client:
-
-```text
-    Host *
-        ServerAliveInterval 120
-```
-
-This should make the client keep the connection alive while validating
-backups.
-
-
-### libyaml compilation errors
-
-borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
-use a C YAML library (libyaml) if present. But if it's not installed, then
-when installing or upgrading borgmatic, you may see errors about compiling the
-YAML library. If so, not to worry. borgmatic should install and function
-correctly even without the C YAML library. And borgmatic won't be any faster
-with the C library present, so you don't need to go out of your way to install
-it.
-
-
-## Issues and feedback
-
-Got an issue or an idea for a feature enhancement? Check out the [borgmatic
-issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues?page=1&status=399951,399952,399955). In
-order to create a new issue or comment on an issue, you'll need to [login
-first](https://tree.taiga.io/login).
+If you'd like to chat with borgmatic developers or users, head on over to the
+`#borgmatic` IRC channel on Freenode, either via <a
+href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
+native <a href="irc://chat.freenode.net:6697">IRC client</a>.
 
 Other questions or comments? Contact <mailto:witten@torsion.org>.
+
+
+### Contributing
+
+If you'd like to contribute to borgmatic development, please feel free to
+submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls)
+or open an [issue](https://projects.torsion.org/witten/borgmatic/issues) first
+to discuss your idea. We also accept Pull Requests on GitHub, if that's more
+your thing. In general, contributions are very welcome. We don't bite! 
+
+Also, please check out the [borgmatic development
+how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
+info on cloning source code, running tests, etc.

borgmatic/borg/check.py

@@ -1,16 +1,19 @@
-import os
-import subprocess
+import logging
 
 from borgmatic.borg import extract
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
+from borgmatic.execute import execute_command
 
 DEFAULT_CHECKS = ('repository', 'archives')
+DEFAULT_PREFIX = '{hostname}-'
 
 
-def _parse_checks(consistency_config):
+logger = logging.getLogger(__name__)
+
+
+def _parse_checks(consistency_config, only_checks=None):
     '''
-    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
+    Given a consistency config with a "checks" list, and an optional list of override checks,
+    transform them a tuple of named checks to run.
 
     For example, given a retention config of:
 
@@ -20,17 +23,24 @@ def _parse_checks(consistency_config):
 
         ('repository', 'archives')
 
-    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
-    "disabled", return an empty tuple, meaning that no checks should be run.
+    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If the checks value
+    is the string "disabled", return an empty tuple, meaning that no checks should be run.
+
+    If the "data" option is present, then make sure the "archives" option is included as well.
     '''
-    checks = consistency_config.get('checks', [])
+    checks = [
+        check.lower() for check in (only_checks or consistency_config.get('checks', []) or [])
+    ]
     if checks == ['disabled']:
         return ()
 
-    return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
+    if 'data' in checks and 'archives' not in checks:
+        checks.append('archives')
+
+    return tuple(check for check in checks if check not in ('disabled', '')) or DEFAULT_CHECKS
 
 
-def _make_check_flags(checks, check_last=None):
+def _make_check_flags(checks, check_last=None, prefix=None):
     '''
     Given a parsed sequence of checks, transform it into tuple of command-line flags.
 
@@ -42,44 +52,81 @@ def _make_check_flags(checks, check_last=None):
 
         ('--repository-only',)
 
-    Additionally, if a check_last value is given, a "--last" flag will be added.
+    However, if both "repository" and "archives" are in checks, then omit them from the returned
+    flags because Borg does both checks by default.
+
+    Additionally, if a check_last value is given and "archives" is in checks, then include a
+    "--last" flag. And if a prefix value is given and "archives" is in checks, then include a
+    "--prefix" flag.
     '''
-    last_flag = ('--last', str(check_last)) if check_last else ()
-    if checks == DEFAULT_CHECKS:
-        return last_flag
+    if 'archives' in checks:
+        last_flags = ('--last', str(check_last)) if check_last else ()
+        prefix_flags = ('--prefix', prefix) if prefix else ()
+    else:
+        last_flags = ()
+        prefix_flags = ()
+        if check_last:
+            logger.warning(
+                'Ignoring check_last option, as "archives" is not in consistency checks.'
+            )
+        if prefix:
+            logger.warning(
+                'Ignoring consistency prefix option, as "archives" is not in consistency checks.'
+            )
 
-    return tuple(
-        '--{}-only'.format(check) for check in checks
-        if check in DEFAULT_CHECKS
-    ) + last_flag
+    common_flags = last_flags + prefix_flags + (('--verify-data',) if 'data' in checks else ())
+
+    if set(DEFAULT_CHECKS).issubset(set(checks)):
+        return common_flags
+
+    return (
+        tuple('--{}-only'.format(check) for check in checks if check in DEFAULT_CHECKS)
+        + common_flags
+    )
 
 
-def check_archives(verbosity, repository, consistency_config, remote_path=None):
+def check_archives(
+    repository,
+    storage_config,
+    consistency_config,
+    local_path='borg',
+    remote_path=None,
+    only_checks=None,
+):
     '''
-    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
-    command to run, check the contained Borg archives for consistency.
+    Given a local or remote repository path, a storage config dict, a consistency config dict,
+    local/remote commands to run, and an optional list of checks to use instead of configured
+    checks, check the contained Borg archives for consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
-    checks = _parse_checks(consistency_config)
+    checks = _parse_checks(consistency_config, only_checks)
     check_last = consistency_config.get('check_last', None)
+    lock_wait = None
 
-    if set(checks).intersection(set(DEFAULT_CHECKS)):
+    if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
         remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-        verbosity_flags = {
-            VERBOSITY_SOME: ('--info',),
-            VERBOSITY_LOTS: ('--debug',),
-        }.get(verbosity, ())
+        lock_wait = storage_config.get('lock_wait', None)
+        lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
+
+        verbosity_flags = ()
+        if logger.isEnabledFor(logging.INFO):
+            verbosity_flags = ('--info',)
+        if logger.isEnabledFor(logging.DEBUG):
+            verbosity_flags = ('--debug', '--show-rc')
+
+        prefix = consistency_config.get('prefix', DEFAULT_PREFIX)
 
         full_command = (
-            'borg', 'check',
-            repository,
-        ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
+            (local_path, 'check')
+            + _make_check_flags(checks, check_last, prefix)
+            + remote_path_flags
+            + lock_wait_flags
+            + verbosity_flags
+            + (repository,)
+        )
 
-        # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
-        stdout = None if verbosity_flags else open(os.devnull, 'w')
-
-        subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
+        execute_command(full_command)
 
     if 'extract' in checks:
-        extract.extract_last_archive_dry_run(verbosity, repository, remote_path)
+        extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -1,17 +1,12 @@
 import glob
 import itertools
+import logging
 import os
-import subprocess
 import tempfile
 
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+from borgmatic.execute import execute_command, execute_command_without_capture
 
-
-def initialize(storage_config):
-    passphrase = storage_config.get('encryption_passphrase')
-
-    if passphrase:
-        os.environ['BORG_PASSPHRASE'] = passphrase
+logger = logging.getLogger(__name__)
 
 
 def _expand_directory(directory):
@@ -24,33 +19,72 @@ def _expand_directory(directory):
     return glob.glob(expanded_directory) or [expanded_directory]
 
 
-def _write_exclude_file(exclude_patterns=None):
+def _expand_directories(directories):
     '''
-    Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
-    None if no patterns are provided.
+    Given a sequence of directory paths, expand tildes and globs in each one. Return all the
+    resulting directories as a single flattened tuple.
     '''
-    if not exclude_patterns:
+    if directories is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(_expand_directory(directory) for directory in directories)
+    )
+
+
+def _expand_home_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
+    Return the results as a tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(os.path.expanduser(directory) for directory in directories)
+
+
+def _write_pattern_file(patterns=None):
+    '''
+    Given a sequence of patterns, write them to a named temporary file and return it. Return None
+    if no patterns are provided.
+    '''
+    if not patterns:
         return None
 
-    exclude_file = tempfile.NamedTemporaryFile('w')
-    exclude_file.write('\n'.join(exclude_patterns))
-    exclude_file.flush()
+    pattern_file = tempfile.NamedTemporaryFile('w')
+    pattern_file.write('\n'.join(patterns))
+    pattern_file.flush()
 
-    return exclude_file
+    return pattern_file
 
 
-def _make_exclude_flags(location_config, exclude_patterns_filename=None):
+def _make_pattern_flags(location_config, pattern_filename=None):
+    '''
+    Given a location config dict with a potential patterns_from option, and a filename containing
+    any additional patterns, return the corresponding Borg flags for those files as a tuple.
+    '''
+    pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
+        (pattern_filename,) if pattern_filename else ()
+    )
+
+    return tuple(
+        itertools.chain.from_iterable(
+            ('--patterns-from', pattern_filename) for pattern_filename in pattern_filenames
+        )
+    )
+
+
+def _make_exclude_flags(location_config, exclude_filename=None):
     '''
     Given a location config dict with various exclude options, and a filename containing any exclude
     patterns, return the corresponding Borg flags as a tuple.
     '''
     exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
-        (exclude_patterns_filename,) if exclude_patterns_filename else ()
+        (exclude_filename,) if exclude_filename else ()
     )
     exclude_from_flags = tuple(
         itertools.chain.from_iterable(
-            ('--exclude-from', exclude_filename)
-            for exclude_filename in exclude_filenames
+            ('--exclude-from', exclude_filename) for exclude_filename in exclude_filenames
         )
     )
     caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
@@ -60,46 +94,102 @@ def _make_exclude_flags(location_config, exclude_patterns_filename=None):
     return exclude_from_flags + caches_flag + if_present_flags
 
 
-def create_archive(
-    verbosity, repository, location_config, storage_config,
-):
+BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
+
+
+def borgmatic_source_directories():
     '''
-    Given a vebosity flag, a local or remote repository path, a location config dict, and a storage
-    config dict, create a Borg archive.
+    Return a list of borgmatic-specific source directories used for state like database backups.
     '''
-    sources = tuple(
-        itertools.chain.from_iterable(
-            _expand_directory(directory)
-            for directory in location_config['source_directories']
-        )
+    return (
+        [BORGMATIC_SOURCE_DIRECTORY]
+        if os.path.exists(os.path.expanduser(BORGMATIC_SOURCE_DIRECTORY))
+        else []
     )
 
-    exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns'))
-    exclude_flags = _make_exclude_flags(
-        location_config,
-        exclude_patterns_file.name if exclude_patterns_file else None,
+
+def create_archive(
+    dry_run,
+    repository,
+    location_config,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+    progress=False,
+    stats=False,
+    json=False,
+):
+    '''
+    Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
+    storage config dict, create a Borg archive and return Borg's JSON output (if any).
+    '''
+    sources = _expand_directories(
+        location_config['source_directories'] + borgmatic_source_directories()
     )
+
+    pattern_file = _write_pattern_file(location_config.get('patterns'))
+    exclude_file = _write_pattern_file(
+        _expand_home_directories(location_config.get('exclude_patterns'))
+    )
+    checkpoint_interval = storage_config.get('checkpoint_interval', None)
+    chunker_params = storage_config.get('chunker_params', None)
     compression = storage_config.get('compression', None)
-    compression_flags = ('--compression', compression) if compression else ()
+    remote_rate_limit = storage_config.get('remote_rate_limit', None)
     umask = storage_config.get('umask', None)
-    umask_flags = ('--umask', str(umask)) if umask else ()
-    one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
-    remote_path = location_config.get('remote_path')
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--list', '--stats'),
-    }.get(verbosity, ())
+    lock_wait = storage_config.get('lock_wait', None)
+    files_cache = location_config.get('files_cache')
     default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
     archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
 
     full_command = (
-        'borg', 'create',
-        '{repository}::{archive_name_format}'.format(
-            repository=repository,
-            archive_name_format=archive_name_format,
-        ),
-    ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
-        remote_path_flags + umask_flags + verbosity_flags
+        (local_path, 'create')
+        + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
+        + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
+        + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
+        + (('--chunker-params', chunker_params) if chunker_params else ())
+        + (('--compression', compression) if compression else ())
+        + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
+        + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--noatime',) if location_config.get('atime') is False else ())
+        + (('--noctime',) if location_config.get('ctime') is False else ())
+        + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
+        + (('--read-special',) if location_config.get('read_special') else ())
+        + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
+        + (('--files-cache', files_cache) if files_cache else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--list', '--filter', 'AME-') if logger.isEnabledFor(logging.INFO) and not json else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
+        + (
+            ('--stats',)
+            if not dry_run and (logger.isEnabledFor(logging.INFO) or stats) and not json
+            else ()
+        )
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
+        + (('--json',) if json else ())
+        + (
+            '{repository}::{archive_name_format}'.format(
+                repository=repository, archive_name_format=archive_name_format
+            ),
+        )
+        + sources
+    )
 
-    subprocess.check_call(full_command)
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if progress:
+        execute_command_without_capture(full_command)
+        return
+
+    if json:
+        output_log_level = None
+    elif stats:
+        output_log_level = logging.WARNING
+    else:
+        output_log_level = logging.INFO
+
+    return execute_command(full_command, output_log_level)

borgmatic/borg/environment.py

@@ -0,0 +1,31 @@
+import os
+
+OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'borg_base_directory': 'BORG_BASE_DIR',
+    'borg_config_directory': 'BORG_CONFIG_DIR',
+    'borg_cache_directory': 'BORG_CACHE_DIR',
+    'borg_security_directory': 'BORG_SECURITY_DIR',
+    'borg_keys_directory': 'BORG_KEYS_DIR',
+    'encryption_passcommand': 'BORG_PASSCOMMAND',
+    'encryption_passphrase': 'BORG_PASSPHRASE',
+    'ssh_command': 'BORG_RSH',
+}
+
+DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
+    'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
+}
+
+
+def initialize(storage_config):
+    for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name)
+        if value:
+            os.environ[environment_variable_name] = value
+
+    for (
+        option_name,
+        environment_variable_name,
+    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name, False)
+        os.environ[environment_variable_name] = 'yes' if value else 'no'

borgmatic/borg/extract.py

@@ -1,40 +1,92 @@
-import sys
-import subprocess
+import logging
 
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+from borgmatic.execute import execute_command, execute_command_without_capture
+
+logger = logging.getLogger(__name__)
 
 
-def extract_last_archive_dry_run(verbosity, repository, remote_path=None):
+def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg', remote_path=None):
     '''
-    Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
-    the dry-run.
+    Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
+    dry-run.
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info',),
-        VERBOSITY_LOTS: ('--debug',),
-    }.get(verbosity, ())
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
+    verbosity_flags = ()
+    if logger.isEnabledFor(logging.DEBUG):
+        verbosity_flags = ('--debug', '--show-rc')
+    elif logger.isEnabledFor(logging.INFO):
+        verbosity_flags = ('--info',)
 
     full_list_command = (
-        'borg', 'list',
-        '--short',
-        repository,
-    ) + remote_path_flags + verbosity_flags
+        (local_path, 'list', '--short')
+        + remote_path_flags
+        + lock_wait_flags
+        + verbosity_flags
+        + (repository,)
+    )
 
-    list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
+    list_output = execute_command(full_list_command, output_log_level=None)
 
-    last_archive_name = list_output.strip().split('\n')[-1]
-    if not last_archive_name:
+    try:
+        last_archive_name = list_output.strip().splitlines()[-1]
+    except IndexError:
         return
 
-    list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
+    list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
     full_extract_command = (
-        'borg', 'extract',
-        '--dry-run',
-        '{repository}::{last_archive_name}'.format(
-            repository=repository,
-            last_archive_name=last_archive_name,
-        ),
-    ) + remote_path_flags + verbosity_flags + list_flag
+        (local_path, 'extract', '--dry-run')
+        + remote_path_flags
+        + lock_wait_flags
+        + verbosity_flags
+        + list_flag
+        + (
+            '{repository}::{last_archive_name}'.format(
+                repository=repository, last_archive_name=last_archive_name
+            ),
+        )
+    )
 
-    subprocess.check_call(full_extract_command)
+    execute_command(full_extract_command)
+
+
+def extract_archive(
+    dry_run,
+    repository,
+    archive,
+    restore_paths,
+    location_config,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+    progress=False,
+):
+    '''
+    Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
+    restore from the archive, and location/storage configuration dicts, extract the archive into the
+    current directory.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'extract')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
+        + ('::'.join((repository, archive)),)
+        + (tuple(restore_paths) if restore_paths else ())
+    )
+
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if progress:
+        execute_command_without_capture(full_command)
+        return
+
+    execute_command(full_command)

borgmatic/borg/flags.py

@@ -0,0 +1,31 @@
+import itertools
+
+
+def make_flags(name, value):
+    '''
+    Given a flag name and its value, return it formatted as Borg-compatible flags.
+    '''
+    if not value:
+        return ()
+
+    flag = '--{}'.format(name.replace('_', '-'))
+
+    if value is True:
+        return (flag,)
+
+    return (flag, str(value))
+
+
+def make_flags_from_arguments(arguments, excludes=()):
+    '''
+    Given borgmatic command-line arguments as an instance of argparse.Namespace, and optionally a
+    list of named arguments to exclude, generate and return the corresponding Borg command-line
+    flags as a tuple.
+    '''
+    return tuple(
+        itertools.chain.from_iterable(
+            make_flags(name, value=getattr(arguments, name))
+            for name in sorted(vars(arguments))
+            if name not in excludes and not name.startswith('_')
+        )
+    )

borgmatic/borg/info.py

@@ -0,0 +1,43 @@
+import logging
+
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def display_archives_info(
+    repository, storage_config, info_arguments, local_path='borg', remote_path=None
+):
+    '''
+    Given a local or remote repository path, a storage config dict, and the arguments to the info
+    action, display summary information for Borg archives in the repository or return JSON summary
+    information.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'info')
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not info_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not info_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(info_arguments, excludes=('repository', 'archive'))
+        + (
+            '::'.join((repository, info_arguments.archive))
+            if info_arguments.archive
+            else repository,
+        )
+    )
+
+    return execute_command(
+        full_command, output_log_level=None if info_arguments.json else logging.WARNING
+    )

borgmatic/borg/init.py

@@ -0,0 +1,48 @@
+import logging
+import subprocess
+
+from borgmatic.execute import execute_command, execute_command_without_capture
+
+logger = logging.getLogger(__name__)
+
+
+INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
+
+
+def initialize_repository(
+    repository,
+    encryption_mode,
+    append_only=None,
+    storage_quota=None,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a Borg encryption mode, whether the repository should
+    be append-only, and the storage quota to use, initialize the repository. If the repository
+    already exists, then log and skip initialization.
+    '''
+    info_command = (local_path, 'info', repository)
+    logger.debug(' '.join(info_command))
+
+    try:
+        execute_command(info_command, output_log_level=None)
+        logger.info('Repository already exists. Skipping initialization.')
+        return
+    except subprocess.CalledProcessError as error:
+        if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
+            raise
+
+    init_command = (
+        (local_path, 'init')
+        + (('--encryption', encryption_mode) if encryption_mode else ())
+        + (('--append-only',) if append_only else ())
+        + (('--storage-quota', storage_quota) if storage_quota else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (repository,)
+    )
+
+    # Don't use execute_command() here because it doesn't support interactive prompts.
+    execute_command_without_capture(init_command)

borgmatic/borg/list.py

@@ -0,0 +1,50 @@
+import logging
+
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+# A hack to convince Borg to exclude archives ending in ".checkpoint". This assumes that a
+# non-checkpoint archive name ends in a digit (e.g. from a timestamp).
+BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[0123456789]'
+
+
+def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
+    '''
+    Given a local or remote repository path, a storage config dict, and the arguments to the list
+    action, display the output of listing Borg archives in the repository or return JSON output. Or,
+    if an archive name is given, listing the files in that archive.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+    if list_arguments.successful:
+        list_arguments.glob_archives = BORG_EXCLUDE_CHECKPOINTS_GLOB
+
+    full_command = (
+        (local_path, 'list')
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(
+            list_arguments, excludes=('repository', 'archive', 'successful')
+        )
+        + (
+            '::'.join((repository, list_arguments.archive))
+            if list_arguments.archive
+            else repository,
+        )
+    )
+
+    return execute_command(
+        full_command, output_log_level=None if list_arguments.json else logging.WARNING
+    )

borgmatic/borg/prune.py

@@ -1,6 +1,8 @@
-import subprocess
+import logging
 
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
 
 
 def _make_prune_flags(retention_config):
@@ -19,33 +21,47 @@ def _make_prune_flags(retention_config):
             ('--keep-monthly', '6'),
         )
     '''
-    if not retention_config.get('prefix'):
-        retention_config['prefix'] = '{hostname}-'
+    config = retention_config.copy()
+
+    if 'prefix' not in config:
+        config['prefix'] = '{hostname}-'
+    elif not config['prefix']:
+        config.pop('prefix')
 
     return (
-        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
-        for option_name, value in retention_config.items()
+        ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
     )
 
 
-def prune_archives(verbosity, repository, retention_config, remote_path=None):
+def prune_archives(
+    dry_run,
+    repository,
+    storage_config,
+    retention_config,
+    local_path='borg',
+    remote_path=None,
+    stats=False,
+):
     '''
-    Given a verbosity flag, a local or remote repository path, a retention config dict, prune Borg
-    archives according the the retention policy specified in that configuration.
+    Given dry-run flag, a local or remote repository path, a storage config dict, and a
+    retention config dict, prune Borg archives according to the retention policy specified in that
+    configuration.
     '''
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
-    }.get(verbosity, ())
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        'borg', 'prune',
-        repository,
-    ) + tuple(
-        element
-        for pair in _make_prune_flags(retention_config)
-        for element in pair
-    ) + remote_path_flags + verbosity_flags
+        (local_path, 'prune')
+        + tuple(element for pair in _make_prune_flags(retention_config) for element in pair)
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--stats',) if not dry_run and logger.isEnabledFor(logging.INFO) else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--stats',) if stats else ())
+        + (repository,)
+    )
 
-    subprocess.check_call(full_command)
+    execute_command(full_command, output_log_level=logging.WARNING if stats else logging.INFO)

borgmatic/commands/arguments.py

@@ -0,0 +1,408 @@
+import collections
+from argparse import ArgumentParser
+
+from borgmatic.config import collect
+
+SUBPARSER_ALIASES = {
+    'init': ['--init', '-I'],
+    'prune': ['--prune', '-p'],
+    'create': ['--create', '-C'],
+    'check': ['--check', '-k'],
+    'extract': ['--extract', '-x'],
+    'list': ['--list', '-l'],
+    'info': ['--info', '-i'],
+}
+
+
+def parse_subparser_arguments(unparsed_arguments, subparsers):
+    '''
+    Given a sequence of arguments, and a subparsers object as returned by
+    argparse.ArgumentParser().add_subparsers(), give each requested action's subparser a shot at
+    parsing all arguments. This allows common arguments like "--repository" to be shared across
+    multiple subparsers.
+
+    Return the result as a dict mapping from subparser name to a parsed namespace of arguments.
+    '''
+    arguments = collections.OrderedDict()
+    remaining_arguments = list(unparsed_arguments)
+    alias_to_subparser_name = {
+        alias: subparser_name
+        for subparser_name, aliases in SUBPARSER_ALIASES.items()
+        for alias in aliases
+    }
+
+    for subparser_name, subparser in subparsers.choices.items():
+        if subparser_name not in remaining_arguments:
+            continue
+
+        canonical_name = alias_to_subparser_name.get(subparser_name, subparser_name)
+
+        # If a parsed value happens to be the same as the name of a subparser, remove it from the
+        # remaining arguments. This prevents, for instance, "check --only extract" from triggering
+        # the "extract" subparser.
+        parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+        for value in vars(parsed).values():
+            if isinstance(value, str):
+                if value in subparsers.choices:
+                    remaining_arguments.remove(value)
+            elif isinstance(value, list):
+                for item in value:
+                    if item in subparsers.choices:
+                        remaining_arguments.remove(item)
+
+        arguments[canonical_name] = parsed
+
+    # If no actions are explicitly requested, assume defaults: prune, create, and check.
+    if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
+        for subparser_name in ('prune', 'create', 'check'):
+            subparser = subparsers.choices[subparser_name]
+            parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+            arguments[subparser_name] = parsed
+
+    return arguments
+
+
+def parse_global_arguments(unparsed_arguments, top_level_parser, subparsers):
+    '''
+    Given a sequence of arguments, a top-level parser (containing subparsers), and a subparsers
+    object as returned by argparse.ArgumentParser().add_subparsers(), parse and return any global
+    arguments as a parsed argparse.Namespace instance.
+    '''
+    # Ask each subparser, one by one, to greedily consume arguments. Any arguments that remain
+    # are global arguments.
+    remaining_arguments = list(unparsed_arguments)
+    present_subparser_names = set()
+
+    for subparser_name, subparser in subparsers.choices.items():
+        if subparser_name not in remaining_arguments:
+            continue
+
+        present_subparser_names.add(subparser_name)
+        unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    # If no actions are explicitly requested, assume defaults: prune, create, and check.
+    if (
+        not present_subparser_names
+        and '--help' not in unparsed_arguments
+        and '-h' not in unparsed_arguments
+    ):
+        for subparser_name in ('prune', 'create', 'check'):
+            subparser = subparsers.choices[subparser_name]
+            unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    # Remove the subparser names themselves.
+    for subparser_name in present_subparser_names:
+        if subparser_name in remaining_arguments:
+            remaining_arguments.remove(subparser_name)
+
+    return top_level_parser.parse_args(remaining_arguments)
+
+
+def parse_arguments(*unparsed_arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    global_parser = ArgumentParser(add_help=False)
+    global_group = global_parser.add_argument_group('global arguments')
+
+    global_group.add_argument(
+        '-c',
+        '--config',
+        nargs='*',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+    global_group.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        help='Deprecated in favor of exclude_patterns within configuration',
+    )
+    global_group.add_argument(
+        '-n',
+        '--dry-run',
+        dest='dry_run',
+        action='store_true',
+        help='Go through the motions, but do not actually write to any repositories',
+    )
+    global_group.add_argument(
+        '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output'
+    )
+    global_group.add_argument(
+        '-v',
+        '--verbosity',
+        type=int,
+        choices=range(0, 3),
+        default=0,
+        help='Display verbose progress to the console (from none to lots: 0, 1, or 2)',
+    )
+    global_group.add_argument(
+        '--syslog-verbosity',
+        type=int,
+        choices=range(0, 3),
+        default=0,
+        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2). Ignored when console is interactive',
+    )
+    global_group.add_argument(
+        '--version',
+        dest='version',
+        default=False,
+        action='store_true',
+        help='Display installed version number of borgmatic and exit',
+    )
+
+    top_level_parser = ArgumentParser(
+        description='''
+            A simple wrapper script for the Borg backup software that creates and prunes backups.
+            If none of the action options are given, then borgmatic defaults to: prune, create, and
+            check archives.
+            ''',
+        parents=[global_parser],
+    )
+
+    subparsers = top_level_parser.add_subparsers(
+        title='actions',
+        metavar='',
+        help='Specify zero or more actions. Defaults to prune, create, and check. Use --help with action for details:',
+    )
+    init_parser = subparsers.add_parser(
+        'init',
+        aliases=SUBPARSER_ALIASES['init'],
+        help='Initialize an empty Borg repository',
+        description='Initialize an empty Borg repository',
+        add_help=False,
+    )
+    init_group = init_parser.add_argument_group('init arguments')
+    init_group.add_argument(
+        '-e',
+        '--encryption',
+        dest='encryption_mode',
+        help='Borg repository encryption mode',
+        required=True,
+    )
+    init_group.add_argument(
+        '--append-only',
+        dest='append_only',
+        action='store_true',
+        help='Create an append-only repository',
+    )
+    init_group.add_argument(
+        '--storage-quota',
+        dest='storage_quota',
+        help='Create a repository with a fixed storage quota',
+    )
+    init_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    prune_parser = subparsers.add_parser(
+        'prune',
+        aliases=SUBPARSER_ALIASES['prune'],
+        help='Prune archives according to the retention policy',
+        description='Prune archives according to the retention policy',
+        add_help=False,
+    )
+    prune_group = prune_parser.add_argument_group('prune arguments')
+    prune_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    create_parser = subparsers.add_parser(
+        'create',
+        aliases=SUBPARSER_ALIASES['create'],
+        help='Create archives (actually perform backups)',
+        description='Create archives (actually perform backups)',
+        add_help=False,
+    )
+    create_group = create_parser.add_argument_group('create arguments')
+    create_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    create_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    create_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    check_parser = subparsers.add_parser(
+        'check',
+        aliases=SUBPARSER_ALIASES['check'],
+        help='Check archives for consistency',
+        description='Check archives for consistency',
+        add_help=False,
+    )
+    check_group = check_parser.add_argument_group('check arguments')
+    check_group.add_argument(
+        '--only',
+        metavar='CHECK',
+        choices=('repository', 'archives', 'data', 'extract'),
+        dest='only',
+        action='append',
+        help='Run a particular consistency check (repository, archives, data, or extract) instead of configured checks; can specify flag multiple times',
+    )
+    check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    extract_parser = subparsers.add_parser(
+        'extract',
+        aliases=SUBPARSER_ALIASES['extract'],
+        help='Extract a named archive to the current directory',
+        description='Extract a named archive to the current directory',
+        add_help=False,
+    )
+    extract_group = extract_parser.add_argument_group('extract arguments')
+    extract_group.add_argument(
+        '--repository',
+        help='Path of repository to extract, defaults to the configured repository if there is only one',
+    )
+    extract_group.add_argument('--archive', help='Name of archive to operate on', required=True)
+    extract_group.add_argument(
+        '--restore-path',
+        nargs='+',
+        dest='restore_paths',
+        help='Paths to restore from archive, defaults to the entire archive',
+    )
+    extract_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    extract_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    list_parser = subparsers.add_parser(
+        'list',
+        aliases=SUBPARSER_ALIASES['list'],
+        help='List archives',
+        description='List archives or the contents of an archive',
+        add_help=False,
+    )
+    list_group = list_parser.add_argument_group('list arguments')
+    list_group.add_argument(
+        '--repository',
+        help='Path of repository to list, defaults to the configured repository if there is only one',
+    )
+    list_group.add_argument('--archive', help='Name of archive to list')
+    list_group.add_argument(
+        '--short', default=False, action='store_true', help='Output only archive or path names'
+    )
+    list_group.add_argument('--format', help='Format for file listing')
+    list_group.add_argument(
+        '--json', default=False, action='store_true', help='Output results as JSON'
+    )
+    list_group.add_argument(
+        '-P', '--prefix', help='Only list archive names starting with this prefix'
+    )
+    list_group.add_argument(
+        '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob'
+    )
+    list_group.add_argument(
+        '--successful',
+        default=False,
+        action='store_true',
+        help='Only list archive names of successful (non-checkpoint) backups',
+    )
+    list_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    list_group.add_argument(
+        '--first', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '--last', metavar='N', help='List last N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
+    )
+    list_group.add_argument(
+        '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
+    )
+    list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
+    list_group.add_argument(
+        '--patterns-from',
+        metavar='FILENAME',
+        help='Include or exclude paths matching patterns from pattern file, one per line',
+    )
+    list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    info_parser = subparsers.add_parser(
+        'info',
+        aliases=SUBPARSER_ALIASES['info'],
+        help='Display summary information on archives',
+        description='Display summary information on archives',
+        add_help=False,
+    )
+    info_group = info_parser.add_argument_group('info arguments')
+    info_group.add_argument(
+        '--repository',
+        help='Path of repository to show info for, defaults to the configured repository if there is only one',
+    )
+    info_group.add_argument('--archive', help='Name of archive to show info for')
+    info_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    info_group.add_argument(
+        '-P', '--prefix', help='Only show info for archive names starting with this prefix'
+    )
+    info_group.add_argument(
+        '-a',
+        '--glob-archives',
+        metavar='GLOB',
+        help='Only show info for archive names matching this glob',
+    )
+    info_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    info_group.add_argument(
+        '--first',
+        metavar='N',
+        help='Show info for first N archives after other filters are applied',
+    )
+    info_group.add_argument(
+        '--last', metavar='N', help='Show info for first N archives after other filters are applied'
+    )
+    info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    arguments = parse_subparser_arguments(unparsed_arguments, subparsers)
+    arguments['global'] = parse_global_arguments(unparsed_arguments, top_level_parser, subparsers)
+
+    if arguments['global'].excludes_filename:
+        raise ValueError(
+            'The --excludes option has been replaced with exclude_patterns in configuration'
+        )
+
+    if 'init' in arguments and arguments['global'].dry_run:
+        raise ValueError('The init action cannot be used with the --dry-run option')
+
+    if 'list' in arguments and arguments['list'].glob_archives and arguments['list'].successful:
+        raise ValueError('The --glob-archives and --successful options cannot be used together')
+
+    if (
+        'list' in arguments
+        and 'info' in arguments
+        and arguments['list'].json
+        and arguments['info'].json
+    ):
+        raise ValueError('With the --json option, list and info actions cannot be used together')
+
+    return arguments

borgmatic/commands/borgmatic.py

@@ -1,137 +1,452 @@
-
-from argparse import ArgumentParser
+import collections
+import json
 import logging
 import os
-from subprocess import CalledProcessError
 import sys
+from subprocess import CalledProcessError
 
-from borgmatic.borg import check, create, prune
-from borgmatic.commands import hook
-from borgmatic.config import collect, convert, validate
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS, verbosity_to_log_level
+import colorama
+import pkg_resources
 
+from borgmatic.borg import check as borg_check
+from borgmatic.borg import create as borg_create
+from borgmatic.borg import environment as borg_environment
+from borgmatic.borg import extract as borg_extract
+from borgmatic.borg import info as borg_info
+from borgmatic.borg import init as borg_init
+from borgmatic.borg import list as borg_list
+from borgmatic.borg import prune as borg_prune
+from borgmatic.commands.arguments import parse_arguments
+from borgmatic.config import checks, collect, convert, validate
+from borgmatic.hooks import command, healthchecks, postgresql
+from borgmatic.logger import configure_logging, should_do_markup
+from borgmatic.signals import configure_signals
+from borgmatic.verbosity import verbosity_to_log_level
 
 logger = logging.getLogger(__name__)
 
-
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
 
 
-def parse_arguments(*arguments):
+def run_configuration(config_filename, config, arguments):
     '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
+    Given a config filename, the corresponding parsed config dict, and command-line arguments as a
+    dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
+    backups, consistency checks, and/or other actions.
+
+    Yield a combination of:
+
+      * JSON output strings from successfully executing any actions that produce JSON
+      * logging.LogRecord instances containing errors from any actions or backup hooks that fail
     '''
-    parser = ArgumentParser(
-        description=
-            '''
-            A simple wrapper script for the Borg backup software that creates and prunes backups.
-            If none of the --prune, --create, or --check options are given, then borgmatic defaults
-            to all three: prune, create, and check archives.
-            '''
-    )
-    parser.add_argument(
-        '-c', '--config',
-        nargs='+',
-        dest='config_paths',
-        default=collect.DEFAULT_CONFIG_PATHS,
-        help='Configuration filenames or directories, defaults to: {}'.format(' '.join(collect.DEFAULT_CONFIG_PATHS)),
-    )
-    parser.add_argument(
-        '--excludes',
-        dest='excludes_filename',
-        help='Deprecated in favor of exclude_patterns within configuration',
-    )
-    parser.add_argument(
-        '-p', '--prune',
-        dest='prune',
-        action='store_true',
-        help='Prune archives according to the retention policy',
-    )
-    parser.add_argument(
-        '-C', '--create',
-        dest='create',
-        action='store_true',
-        help='Create archives (actually perform backups)',
-    )
-    parser.add_argument(
-        '-k', '--check',
-        dest='check',
-        action='store_true',
-        help='Check archives for consistency',
-    )
-    parser.add_argument(
-        '-v', '--verbosity',
-        type=int,
-        help='Display verbose progress (1 for some, 2 for lots)',
-    )
-
-    args = parser.parse_args(arguments)
-
-    # If any of the three action flags in the given parse arguments have been explicitly requested,
-    # leave them as-is. Otherwise, assume defaults: Mutate the given arguments to enable all the
-    # actions.
-    if not args.prune and not args.create and not args.check:
-        args.prune = True
-        args.create = True
-        args.check = True
-
-    return args
-
-
-def run_configuration(config_filename, args):  # pragma: no cover
-    '''
-    Parse a single configuration file, and execute its defined pruning, backups, and/or consistency
-    checks.
-    '''
-    logger.info('{}: Parsing configuration file'.format(config_filename))
-    config = validate.parse_configuration(config_filename, validate.schema_filename())
     (location, storage, retention, consistency, hooks) = (
         config.get(section_name, {})
         for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
     )
+    global_arguments = arguments['global']
+
+    local_path = location.get('local_path', 'borg')
+    remote_path = location.get('remote_path')
+    borg_environment.initialize(storage)
+    encountered_error = None
+    error_repository = ''
+
+    if 'create' in arguments:
+        try:
+            command.execute_hook(
+                hooks.get('before_backup'),
+                hooks.get('umask'),
+                config_filename,
+                'pre-backup',
+                global_arguments.dry_run,
+            )
+            postgresql.dump_databases(
+                hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run
+            )
+            healthchecks.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
+            )
+        except (OSError, CalledProcessError) as error:
+            encountered_error = error
+            yield from make_error_log_records(
+                '{}: Error running pre-backup hook'.format(config_filename), error
+            )
+
+    if not encountered_error:
+        for repository_path in location['repositories']:
+            try:
+                yield from run_actions(
+                    arguments=arguments,
+                    location=location,
+                    storage=storage,
+                    retention=retention,
+                    consistency=consistency,
+                    local_path=local_path,
+                    remote_path=remote_path,
+                    repository_path=repository_path,
+                )
+            except (OSError, CalledProcessError) as error:
+                encountered_error = error
+                error_repository = repository_path
+                yield from make_error_log_records(
+                    '{}: Error running actions for repository'.format(repository_path), error
+                )
+
+    if 'create' in arguments and not encountered_error:
+        try:
+            command.execute_hook(
+                hooks.get('after_backup'),
+                hooks.get('umask'),
+                config_filename,
+                'post-backup',
+                global_arguments.dry_run,
+            )
+            postgresql.remove_database_dumps(
+                hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run
+            )
+            healthchecks.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run
+            )
+        except (OSError, CalledProcessError) as error:
+            encountered_error = error
+            yield from make_error_log_records(
+                '{}: Error running post-backup hook'.format(config_filename), error
+            )
+
+    if encountered_error:
+        try:
+            command.execute_hook(
+                hooks.get('on_error'),
+                hooks.get('umask'),
+                config_filename,
+                'on-error',
+                global_arguments.dry_run,
+                repository=error_repository,
+                error=encountered_error,
+                output=getattr(encountered_error, 'output', ''),
+            )
+            healthchecks.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail'
+            )
+        except (OSError, CalledProcessError) as error:
+            yield from make_error_log_records(
+                '{}: Error running on-error hook'.format(config_filename), error
+            )
+
+
+def run_actions(
+    *,
+    arguments,
+    location,
+    storage,
+    retention,
+    consistency,
+    local_path,
+    remote_path,
+    repository_path
+):  # pragma: no cover
+    '''
+    Given parsed command-line arguments as an argparse.ArgumentParser instance, several different
+    configuration dicts, local and remote paths to Borg, and a repository name, run all actions
+    from the command-line arguments on the given repository.
+
+    Yield JSON output strings from executing any actions that produce JSON.
+    '''
+    repository = os.path.expanduser(repository_path)
+    global_arguments = arguments['global']
+    dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else ''
+    if 'init' in arguments:
+        logger.info('{}: Initializing repository'.format(repository))
+        borg_init.initialize_repository(
+            repository,
+            arguments['init'].encryption_mode,
+            arguments['init'].append_only,
+            arguments['init'].storage_quota,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+    if 'prune' in arguments:
+        logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
+        borg_prune.prune_archives(
+            global_arguments.dry_run,
+            repository,
+            storage,
+            retention,
+            local_path=local_path,
+            remote_path=remote_path,
+            stats=arguments['prune'].stats,
+        )
+    if 'create' in arguments:
+        logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
+        json_output = borg_create.create_archive(
+            global_arguments.dry_run,
+            repository,
+            location,
+            storage,
+            local_path=local_path,
+            remote_path=remote_path,
+            progress=arguments['create'].progress,
+            stats=arguments['create'].stats,
+            json=arguments['create'].json,
+        )
+        if json_output:
+            yield json.loads(json_output)
+    if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
+        logger.info('{}: Running consistency checks'.format(repository))
+        borg_check.check_archives(
+            repository,
+            storage,
+            consistency,
+            local_path=local_path,
+            remote_path=remote_path,
+            only_checks=arguments['check'].only,
+        )
+    if 'extract' in arguments:
+        if arguments['extract'].repository is None or repository == arguments['extract'].repository:
+            logger.info(
+                '{}: Extracting archive {}'.format(repository, arguments['extract'].archive)
+            )
+            borg_extract.extract_archive(
+                global_arguments.dry_run,
+                repository,
+                arguments['extract'].archive,
+                arguments['extract'].restore_paths,
+                location,
+                storage,
+                local_path=local_path,
+                remote_path=remote_path,
+                progress=arguments['extract'].progress,
+            )
+    if 'list' in arguments:
+        if arguments['list'].repository is None or repository == arguments['list'].repository:
+            logger.info('{}: Listing archives'.format(repository))
+            json_output = borg_list.list_archives(
+                repository,
+                storage,
+                list_arguments=arguments['list'],
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+            if json_output:
+                yield json.loads(json_output)
+    if 'info' in arguments:
+        if arguments['info'].repository is None or repository == arguments['info'].repository:
+            logger.info('{}: Displaying summary info for archives'.format(repository))
+            json_output = borg_info.display_archives_info(
+                repository,
+                storage,
+                info_arguments=arguments['info'],
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+            if json_output:
+                yield json.loads(json_output)
+
+
+def load_configurations(config_filenames):
+    '''
+    Given a sequence of configuration filenames, load and validate each configuration file. Return
+    the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
+    and sequence of logging.LogRecord instances containing any parse errors.
+    '''
+    # Dict mapping from config filename to corresponding parsed config dict.
+    configs = collections.OrderedDict()
+    logs = []
+
+    # Parse and load each configuration file.
+    for config_filename in config_filenames:
+        try:
+            configs[config_filename] = validate.parse_configuration(
+                config_filename, validate.schema_filename()
+            )
+        except (ValueError, OSError, validate.Validation_error) as error:
+            logs.extend(
+                [
+                    logging.makeLogRecord(
+                        dict(
+                            levelno=logging.CRITICAL,
+                            levelname='CRITICAL',
+                            msg='{}: Error parsing configuration file'.format(config_filename),
+                        )
+                    ),
+                    logging.makeLogRecord(
+                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+                    ),
+                ]
+            )
+
+    return (configs, logs)
+
+
+def make_error_log_records(message, error=None):
+    '''
+    Given error message text and an optional exception object, yield a series of logging.LogRecord
+    instances with error summary information.
+    '''
+    if not error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        return
 
     try:
-        remote_path = location.get('remote_path')
-        create.initialize(storage)
-        hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
+        raise error
+    except CalledProcessError as error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
+        )
+        yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
+    except (ValueError, OSError) as error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
+    except:  # noqa: E722
+        # Raising above only as a means of determining the error type. Swallow the exception here
+        # because we don't want the exception to propagate out of this function.
+        pass
 
-        for repository in location['repositories']:
-            if args.prune:
-                logger.info('{}: Pruning archives'.format(repository))
-                prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
-            if args.create:
-                logger.info('{}: Creating archive'.format(repository))
-                create.create_archive(
-                    args.verbosity,
-                    repository,
-                    location,
-                    storage,
+
+def collect_configuration_run_summary_logs(configs, arguments):
+    '''
+    Given a dict of configuration filename to corresponding parsed configuration, and parsed
+    command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
+    each configuration file and yield a series of logging.LogRecord instances containing summary
+    information about each run.
+
+    As a side effect of running through these configuration files, output their JSON results, if
+    any, to stdout.
+    '''
+    # Run cross-file validation checks.
+    if 'extract' in arguments:
+        repository = arguments['extract'].repository
+    elif 'list' in arguments and arguments['list'].archive:
+        repository = arguments['list'].repository
+    else:
+        repository = None
+
+    if repository:
+        try:
+            validate.guard_configuration_contains_repository(repository, configs)
+        except ValueError as error:
+            yield from make_error_log_records(str(error))
+            return
+
+    if not configs:
+        yield from make_error_log_records(
+            '{}: No configuration files found'.format(' '.join(arguments['global'].config_paths))
+        )
+        return
+
+    if 'create' in arguments:
+        try:
+            for config_filename, config in configs.items():
+                hooks = config.get('hooks', {})
+                command.execute_hook(
+                    hooks.get('before_everything'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'pre-everything',
+                    arguments['global'].dry_run,
                 )
-            if args.check:
-                logger.info('{}: Running consistency checks'.format(repository))
-                check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
+        except (CalledProcessError, ValueError, OSError) as error:
+            yield from make_error_log_records('Error running pre-everything hook', error)
+            return
 
-        hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
-    except (OSError, CalledProcessError):
-        hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error')
-        raise
+    # Execute the actions corresponding to each configuration file.
+    json_results = []
+    for config_filename, config in configs.items():
+        results = list(run_configuration(config_filename, config, arguments))
+        error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord))
+
+        if error_logs:
+            yield from make_error_log_records(
+                '{}: Error running configuration file'.format(config_filename)
+            )
+            yield from error_logs
+        else:
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.INFO,
+                    levelname='INFO',
+                    msg='{}: Successfully ran configuration file'.format(config_filename),
+                )
+            )
+            if results:
+                json_results.extend(results)
+
+    if json_results:
+        sys.stdout.write(json.dumps(json_results))
+
+    if 'create' in arguments:
+        try:
+            for config_filename, config in configs.items():
+                hooks = config.get('hooks', {})
+                command.execute_hook(
+                    hooks.get('after_everything'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'post-everything',
+                    arguments['global'].dry_run,
+                )
+        except (CalledProcessError, ValueError, OSError) as error:
+            yield from make_error_log_records('Error running post-everything hook', error)
+
+
+def exit_with_help_link():  # pragma: no cover
+    '''
+    Display a link to get help and exit with an error code.
+    '''
+    logger.critical('')
+    logger.critical('Need some help? https://torsion.org/borgmatic/#issues')
+    sys.exit(1)
 
 
 def main():  # pragma: no cover
+    configure_signals()
+
     try:
-        args = parse_arguments(*sys.argv[1:])
-        logging.basicConfig(level=verbosity_to_log_level(args.verbosity), format='%(message)s')
+        arguments = parse_arguments(*sys.argv[1:])
+    except ValueError as error:
+        configure_logging(logging.CRITICAL)
+        logger.critical(error)
+        exit_with_help_link()
+    except SystemExit as error:
+        if error.code == 0:
+            raise error
+        configure_logging(logging.CRITICAL)
+        logger.critical('Error parsing arguments: {}'.format(' '.join(sys.argv)))
+        exit_with_help_link()
 
-        config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
-        logger.debug('Ensuring legacy configuration is upgraded')
-        convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
+    global_arguments = arguments['global']
+    if global_arguments.version:
+        print(pkg_resources.require('borgmatic')[0].version)
+        sys.exit(0)
 
-        if len(config_filenames) == 0:
-            raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths)))
+    config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
+    configs, parse_logs = load_configurations(config_filenames)
 
-        for config_filename in config_filenames:
-            run_configuration(config_filename, args)
-    except (ValueError, OSError, CalledProcessError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)
+    colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
+    configure_logging(
+        verbosity_to_log_level(global_arguments.verbosity),
+        verbosity_to_log_level(global_arguments.syslog_verbosity),
+    )
+
+    logger.debug('Ensuring legacy configuration is upgraded')
+    convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
+
+    summary_logs = list(collect_configuration_run_summary_logs(configs, arguments))
+
+    logger.info('')
+    logger.info('summary:')
+    [
+        logger.handle(log)
+        for log in parse_logs + summary_logs
+        if log.levelno >= logger.getEffectiveLevel()
+    ]
+
+    if any(log.levelno == logging.CRITICAL for log in summary_logs):
+        exit_with_help_link()

borgmatic/commands/convert_config.py

@@ -1,14 +1,12 @@
-from argparse import ArgumentParser
 import os
-from subprocess import CalledProcessError
 import sys
 import textwrap
+from argparse import ArgumentParser
 
 from ruamel import yaml
 
 from borgmatic.config import convert, generate, legacy, validate
 
-
 DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
 DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
@@ -26,22 +24,31 @@ def parse_arguments(*arguments):
         '''
     )
     parser.add_argument(
-        '-s', '--source-config',
+        '-s',
+        '--source-config',
         dest='source_config_filename',
         default=DEFAULT_SOURCE_CONFIG_FILENAME,
-        help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
+        help='Source INI-style configuration filename. Default: {}'.format(
+            DEFAULT_SOURCE_CONFIG_FILENAME
+        ),
     )
     parser.add_argument(
-        '-e', '--source-excludes',
+        '-e',
+        '--source-excludes',
         dest='source_excludes_filename',
-        default=DEFAULT_SOURCE_EXCLUDES_FILENAME if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) else None,
+        default=DEFAULT_SOURCE_EXCLUDES_FILENAME
+        if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME)
+        else None,
         help='Excludes filename',
     )
     parser.add_argument(
-        '-d', '--destination-config',
+        '-d',
+        '--destination-config',
         dest='destination_config_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
+        help='Destination YAML configuration filename. Default: {}'.format(
+            DEFAULT_DESTINATION_CONFIG_FILENAME
+        ),
     )
 
     return parser.parse_args(arguments)
@@ -57,11 +64,13 @@ def display_result(args):  # pragma: no cover
         ),
         TEXT_WRAP_CHARACTERS,
     )
-    
+
     delete_lines = textwrap.wrap(
         'Once you are satisfied, you can safely delete {}{}.'.format(
             args.source_config_filename,
-            ' and {}'.format(args.source_excludes_filename) if args.source_excludes_filename else '',
+            ' and {}'.format(args.source_excludes_filename)
+            if args.source_excludes_filename
+            else '',
         ),
         TEXT_WRAP_CHARACTERS,
     )
@@ -75,7 +84,9 @@ def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
         schema = yaml.round_trip_load(open(validate.schema_filename()).read())
-        source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT)
+        source_config = legacy.parse_configuration(
+            args.source_config_filename, legacy.CONFIG_FORMAT
+        )
         source_config_file_mode = os.stat(args.source_config_filename).st_mode
         source_excludes = (
             open(args.source_excludes_filename).read().splitlines()
@@ -83,12 +94,12 @@ def main():  # pragma: no cover
             else []
         )
 
-        destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema)
+        destination_config = convert.convert_legacy_parsed_config(
+            source_config, source_excludes, schema
+        )
 
         generate.write_configuration(
-            args.destination_config_filename,
-            destination_config,
-            mode=source_config_file_mode,
+            args.destination_config_filename, destination_config, mode=source_config_file_mode
         )
 
         display_result(args)

borgmatic/commands/generate_config.py

@@ -1,10 +1,7 @@
-from argparse import ArgumentParser
-import os
-from subprocess import CalledProcessError
 import sys
+from argparse import ArgumentParser
 
-from borgmatic.config import convert, generate, validate
-
+from borgmatic.config import generate, validate
 
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 
@@ -16,10 +13,13 @@ def parse_arguments(*arguments):
     '''
     parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
     parser.add_argument(
-        '-d', '--destination',
+        '-d',
+        '--destination',
         dest='destination_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
+        help='Destination YAML configuration filename. Default: {}'.format(
+            DEFAULT_DESTINATION_CONFIG_FILENAME
+        ),
     )
 
     return parser.parse_args(arguments)
@@ -29,12 +29,16 @@ def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
 
-        generate.generate_sample_configuration(args.destination_filename, validate.schema_filename())
+        generate.generate_sample_configuration(
+            args.destination_filename, validate.schema_filename()
+        )
 
         print('Generated a sample configuration file at {}.'.format(args.destination_filename))
         print()
-        print('Please edit the file to suit your needs. The values are just representative.')
+        print('Please edit the file to suit your needs. The values are representative.')
         print('All fields are optional except where indicated.')
+        print()
+        print('If you ever need help: https://torsion.org/borgmatic/#issues')
     except (ValueError, OSError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

borgmatic/commands/hook.py

@@ -1,20 +0,0 @@
-import logging
-import subprocess
-
-
-logger = logging.getLogger(__name__)
-
-
-def execute_hook(commands, config_filename, description):
-    if not commands:
-        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
-        return
-
-    if len(commands) == 1:
-        logger.info('{}: Running command for {} hook'.format(config_filename, description))
-    else:
-        logger.info('{}: Running {} commands for {} hook'.format(config_filename, len(commands), description))
-
-    for command in commands:
-        logger.debug('{}: Hook command: {}'.format(config_filename, command))
-        subprocess.check_call(command, shell=True)

borgmatic/commands/validate_config.py

@@ -0,0 +1,56 @@
+import logging
+import sys
+from argparse import ArgumentParser
+
+from borgmatic.config import collect, validate
+
+logger = logging.getLogger(__name__)
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    parser = ArgumentParser(description='Validate borgmatic configuration file(s).')
+    parser.add_argument(
+        '-c',
+        '--config',
+        nargs='+',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():  # pragma: no cover
+    args = parse_arguments(*sys.argv[1:])
+
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    if len(config_filenames) == 0:
+        logger.critical('No files to validate found')
+        sys.exit(1)
+
+    found_issues = False
+    for config_filename in config_filenames:
+        try:
+            validate.parse_configuration(config_filename, validate.schema_filename())
+        except (ValueError, OSError, validate.Validation_error) as error:
+            logging.critical('{}: Error parsing configuration file'.format(config_filename))
+            logging.critical(error)
+            found_issues = True
+
+    if found_issues:
+        sys.exit(1)
+    else:
+        logger.info(
+            'All given configuration files are valid: {}'.format(', '.join(config_filenames))
+        )

borgmatic/config/checks.py

@@ -0,0 +1,9 @@
+def repository_enabled_for_checks(repository, consistency):
+    '''
+    Given a repository name and a consistency configuration dict, return whether the repository
+    is enabled to have consistency checks run.
+    '''
+    if not consistency.get('check_repositories'):
+        return True
+
+    return repository in consistency['check_repositories']

borgmatic/config/collect.py

@@ -1,30 +1,48 @@
 import os
 
 
-DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d']
+def get_default_config_paths():
+    '''
+    Based on the value of the XDG_CONFIG_HOME and HOME environment variables, return a list of
+    default configuration paths. This includes both system-wide configuration and configuration in
+    the current user's home directory.
+    '''
+    user_config_directory = os.getenv('XDG_CONFIG_HOME') or os.path.expandvars(
+        os.path.join('$HOME', '.config')
+    )
+
+    return [
+        '/etc/borgmatic/config.yaml',
+        '/etc/borgmatic.d',
+        '%s/borgmatic/config.yaml' % user_config_directory,
+    ]
 
 
 def collect_config_filenames(config_paths):
     '''
-    Given a sequence of config paths, both filenames and directories, resolve that to just an
-    iterable of files. Accomplish this by listing any given directories looking for contained config
-    files. This is non-recursive, so any directories within the given directories are ignored.
+    Given a sequence of config paths, both filenames and directories, resolve that to an iterable
+    of files. Accomplish this by listing any given directories looking for contained config files
+    (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories within the given
+    directories are ignored.
 
     Return paths even if they don't exist on disk, so the user can find out about missing
-    configuration paths. However, skip /etc/borgmatic.d if it's missing, so the user doesn't have to
-    create it unless they need it.
+    configuration paths. However, skip a default config path if it's missing, so the user doesn't
+    have to create a default config path unless they need it.
     '''
+    real_default_config_paths = set(map(os.path.realpath, get_default_config_paths()))
+
     for path in config_paths:
         exists = os.path.exists(path)
 
-        if os.path.realpath(path) in DEFAULT_CONFIG_PATHS and not exists:
+        if os.path.realpath(path) in real_default_config_paths and not exists:
             continue
 
         if not os.path.isdir(path) or not exists:
             yield path
             continue
 
-        for filename in os.listdir(path):
+        for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
-            if not os.path.isdir(full_filename):
+            matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
+            if matching_filetype and not os.path.isdir(full_filename):
                 yield full_filename

borgmatic/config/convert.py

@@ -12,14 +12,17 @@ def _convert_section(source_section_config, section_schema):
 
     Where integer types exist in the given section schema, convert their values to integers.
     '''
-    destination_section_config = yaml.comments.CommentedMap([
-        (
-            option_name,
-            int(option_value)
-                if section_schema['map'].get(option_name, {}).get('type') == 'int' else option_value
-        )
-        for option_name, option_value in source_section_config.items()
-    ])
+    destination_section_config = yaml.comments.CommentedMap(
+        [
+            (
+                option_name,
+                int(option_value)
+                if section_schema['map'].get(option_name, {}).get('type') == 'int'
+                else option_value,
+            )
+            for option_name, option_value in source_section_config.items()
+        ]
+    )
 
     return destination_section_config
 
@@ -33,10 +36,12 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
     Additionally, use the given schema as a source of helpful comments to include within the
     returned CommentedMap.
     '''
-    destination_config = yaml.comments.CommentedMap([
-        (section_name, _convert_section(section_config, schema['map'][section_name]))
-        for section_name, section_config in source_config._asdict().items()
-    ])
+    destination_config = yaml.comments.CommentedMap(
+        [
+            (section_name, _convert_section(section_config, schema['map'][section_name]))
+            for section_name, section_config in source_config._asdict().items()
+        ]
+    )
 
     # Split space-seperated values into actual lists, make "repository" into a list, and merge in
     # excludes.
@@ -49,21 +54,19 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
 
     # Add comments to each section, and then add comments to the fields in each section.
-    generate.add_comments_to_configuration(destination_config, schema)
+    generate.add_comments_to_configuration_map(destination_config, schema)
 
     for section_name, section_config in destination_config.items():
-        generate.add_comments_to_configuration(
-            section_config,
-            schema['map'][section_name],
-            indent=generate.INDENT,
+        generate.add_comments_to_configuration_map(
+            section_config, schema['map'][section_name], indent=generate.INDENT
         )
 
     return destination_config
 
 
-class LegacyConfigurationNotUpgraded(FileNotFoundError):
+class Legacy_configuration_not_upgraded(FileNotFoundError):
     def __init__(self):
-        super(LegacyConfigurationNotUpgraded, self).__init__(
+        super(Legacy_configuration_not_upgraded, self).__init__(
             '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
 to YAML. This better supports validation, and has a more natural way to express
 lists of values. To upgrade your existing configuration, run:
@@ -80,32 +83,13 @@ instead of the old one.'''
 def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
     '''
     If legacy source configuration exists but no destination upgraded configs do, raise
-    LegacyConfigurationNotUpgraded.
+    Legacy_configuration_not_upgraded.
 
     The idea is that we want to alert the user about upgrading their config if they haven't already.
     '''
     destination_config_exists = any(
-        os.path.exists(filename)
-        for filename in destination_config_filenames
+        os.path.exists(filename) for filename in destination_config_filenames
     )
 
     if os.path.exists(source_config_filename) and not destination_config_exists:
-        raise LegacyConfigurationNotUpgraded()
-
-
-class LegacyExcludesFilenamePresent(FileNotFoundError):
-    def __init__(self):
-        super(LegacyExcludesFilenamePresent, self).__init__(
-            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
-to YAML. This better supports validation, and has a more natural way to express
-lists of values. The new configuration file incorporates excludes, so you no
-longer need to provide an excludes filename on the command-line with an
-"--excludes" argument.
-
-Please remove the "--excludes" argument and run borgmatic again.'''
-        )
-
-
-def guard_excludes_filename_omitted(excludes_filename):
-    if excludes_filename != None:
-        raise LegacyExcludesFilenamePresent()
+        raise Legacy_configuration_not_upgraded()

borgmatic/config/generate.py

@@ -1,24 +1,24 @@
-from collections import OrderedDict
+import io
 import os
+import re
 
 from ruamel import yaml
 
-
 INDENT = 4
+SEQUENCE_INDENT = 2
 
 
 def _insert_newline_before_comment(config, field_name):
     '''
-    Using some ruamel.yaml black magic, insert a blank line in the config right befor the given
+    Using some ruamel.yaml black magic, insert a blank line in the config right before the given
     field and its comments.
     '''
     config.ca.items[field_name][1].insert(
-        0,
-        yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None),
+        0, yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None)
     )
 
 
-def _schema_to_sample_configuration(schema, level=0):
+def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     '''
     Given a loaded configuration schema, generate and return sample config for it. Include comments
     for each section based on the schema "desc" description.
@@ -27,23 +27,98 @@ def _schema_to_sample_configuration(schema, level=0):
     if example is not None:
         return example
 
-    config = yaml.comments.CommentedMap([
-        (
-            section_name,
-            _schema_to_sample_configuration(section_schema, level + 1),
+    if 'seq' in schema:
+        config = yaml.comments.CommentedSeq(
+            [
+                _schema_to_sample_configuration(item_schema, level, parent_is_sequence=True)
+                for item_schema in schema['seq']
+            ]
         )
-        for section_name, section_schema in schema['map'].items()
-    ])
-
-    add_comments_to_configuration(config, schema, indent=(level * INDENT))
+        add_comments_to_configuration_sequence(
+            config, schema, indent=(level * INDENT) + SEQUENCE_INDENT
+        )
+    elif 'map' in schema:
+        config = yaml.comments.CommentedMap(
+            [
+                (section_name, _schema_to_sample_configuration(section_schema, level + 1))
+                for section_name, section_schema in schema['map'].items()
+            ]
+        )
+        indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
+        add_comments_to_configuration_map(
+            config, schema, indent=indent, skip_first=parent_is_sequence
+        )
+    else:
+        raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema))
 
     return config
 
 
-def write_configuration(config_filename, config, mode=0o600):
+def _comment_out_line(line):
+    # If it's already is commented out (or empty), there's nothing further to do!
+    stripped_line = line.lstrip()
+    if not stripped_line or stripped_line.startswith('#'):
+        return line
+
+    # Comment out the names of optional sections, inserting the '#' after any indent for aesthetics.
+    matches = re.match(r'(\s*)', line)
+    indent_spaces = matches.group(0) if matches else ''
+    count_indent_spaces = len(indent_spaces)
+
+    return '# '.join((indent_spaces, line[count_indent_spaces:]))
+
+
+REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
+REQUIRED_SECTION_NAMES = {'location', 'retention'}
+
+
+def _comment_out_optional_configuration(rendered_config):
     '''
-    Given a target config filename and a config data structure of nested OrderedDicts, write out the
-    config to file as YAML. Create any containing directories as needed.
+    Post-process a rendered configuration string to comment out optional key/values. The idea is
+    that this prevents the user from having to comment out a bunch of configuration they don't care
+    about to get to a minimal viable configuration file.
+
+    Ideally ruamel.yaml would support this during configuration generation, but it's not terribly
+    easy to accomplish that way.
+    '''
+    lines = []
+    required = False
+
+    for line in rendered_config.split('\n'):
+        key = line.strip().split(':')[0]
+
+        if key in REQUIRED_SECTION_NAMES:
+            lines.append(line)
+            continue
+
+        # Upon encountering a required configuration option, skip commenting out lines until the
+        # next blank line.
+        if key in REQUIRED_KEYS:
+            required = True
+        elif not key:
+            required = False
+
+        lines.append(_comment_out_line(line) if not required else line)
+
+    return '\n'.join(lines)
+
+
+def _render_configuration(config):
+    '''
+    Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
+    '''
+    dumper = yaml.YAML()
+    dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
+    rendered = io.StringIO()
+    dumper.dump(config, rendered)
+
+    return rendered.getvalue()
+
+
+def write_configuration(config_filename, rendered_config, mode=0o600):
+    '''
+    Given a target config filename and rendered config YAML, write it out to file. Create any
+    containing directories as needed.
     '''
     if os.path.exists(config_filename):
         raise FileExistsError('{} already exists. Aborting.'.format(config_filename))
@@ -54,18 +129,54 @@ def write_configuration(config_filename, config, mode=0o600):
         pass
 
     with open(config_filename, 'w') as config_file:
-        config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
+        config_file.write(rendered_config)
 
     os.chmod(config_filename, mode)
 
 
-def add_comments_to_configuration(config, schema, indent=0):
+def add_comments_to_configuration_sequence(config, schema, indent=0):
+    '''
+    If the given config sequence's items are maps, then mine the schema for the description of the
+    map's first item, and slap that atop the sequence. Indent the comment the given number of
+    characters.
+
+    Doing this for sequences of maps results in nice comments that look like:
+
+    ```
+    things:
+          # First key description. Added by this function.
+        - key: foo
+          # Second key description. Added by add_comments_to_configuration_map().
+          other: bar
+    ```
+    '''
+    if 'map' not in schema['seq'][0]:
+        return
+
+    for field_name in config[0].keys():
+        field_schema = schema['seq'][0]['map'].get(field_name, {})
+        description = field_schema.get('desc')
+
+        # No description to use? Skip it.
+        if not field_schema or not description:
+            return
+
+        config[0].yaml_set_start_comment(description, indent=indent)
+
+        # We only want the first key's description here, as the rest of the keys get commented by
+        # add_comments_to_configuration_map().
+        return
+
+
+def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False):
     '''
     Using descriptions from a schema as a source, add those descriptions as comments to the given
-    config before each field. This function only adds comments for the top-most config map level.
-    Indent the comment the given number of characters.
+    config mapping, before each field. Indent the comment the given number of characters.
     '''
     for index, field_name in enumerate(config.keys()):
+        if skip_first and index == 0:
+            continue
+
         field_schema = schema['map'].get(field_name, {})
         description = field_schema.get('desc')
 
@@ -73,11 +184,8 @@ def add_comments_to_configuration(config, schema, indent=0):
         if not field_schema or not description:
             continue
 
-        config.yaml_set_comment_before_after_key(
-            key=field_name,
-            before=description,
-            indent=indent,
-        )
+        config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
+
         if index > 0:
             _insert_newline_before_comment(config, field_name)
 
@@ -90,4 +198,6 @@ def generate_sample_configuration(config_filename, schema_filename):
     schema = yaml.round_trip_load(open(schema_filename))
     config = _schema_to_sample_configuration(schema)
 
-    write_configuration(config_filename, config)
+    write_configuration(
+        config_filename, _comment_out_optional_configuration(_render_configuration(config))
+    )

borgmatic/config/legacy.py

@@ -1,7 +1,6 @@
 from collections import OrderedDict, namedtuple
 from configparser import RawConfigParser
 
-
 Section_format = namedtuple('Section_format', ('name', 'options'))
 Config_option = namedtuple('Config_option', ('name', 'value_type', 'required'))
 
@@ -45,12 +44,8 @@ CONFIG_FORMAT = (
         ),
     ),
     Section_format(
-        'consistency',
-        (
-            option('checks', required=False),
-            option('check_last', required=False),
-        ),
-    )
+        'consistency', (option('checks', required=False), option('check_last', required=False))
+    ),
 )
 
 
@@ -66,7 +61,8 @@ def validate_configuration_format(parser, config_format):
     '''
     section_names = set(parser.sections())
     required_section_names = tuple(
-        section.name for section in config_format
+        section.name
+        for section in config_format
         if any(option.required for option in section.options)
     )
 
@@ -80,9 +76,7 @@ def validate_configuration_format(parser, config_format):
 
     missing_section_names = set(required_section_names) - section_names
     if missing_section_names:
-        raise ValueError(
-            'Missing config sections: {}'.format(', '.join(missing_section_names))
-        )
+        raise ValueError('Missing config sections: {}'.format(', '.join(missing_section_names)))
 
     for section_format in config_format:
         if section_format.name not in section_names:
@@ -91,26 +85,28 @@ def validate_configuration_format(parser, config_format):
         option_names = parser.options(section_format.name)
         expected_options = section_format.options
 
-        unexpected_option_names = set(option_names) - set(option.name for option in expected_options)
+        unexpected_option_names = set(option_names) - set(
+            option.name for option in expected_options
+        )
 
         if unexpected_option_names:
             raise ValueError(
                 'Unexpected options found in config section {}: {}'.format(
-                    section_format.name,
-                    ', '.join(sorted(unexpected_option_names)),
+                    section_format.name, ', '.join(sorted(unexpected_option_names))
                 )
             )
 
         missing_option_names = tuple(
-            option.name for option in expected_options if option.required
+            option.name
+            for option in expected_options
+            if option.required
             if option.name not in option_names
         )
 
         if missing_option_names:
             raise ValueError(
                 'Required options missing from config section {}: {}'.format(
-                    section_format.name,
-                    ', '.join(missing_option_names)
+                    section_format.name, ', '.join(missing_option_names)
                 )
             )
 
@@ -123,11 +119,7 @@ def parse_section_options(parser, section_format):
 
     Raise ValueError if any option values cannot be coerced to the expected Python data type.
     '''
-    type_getter = {
-        str: parser.get,
-        int: parser.getint,
-        bool: parser.getboolean,
-    }
+    type_getter = {str: parser.get, int: parser.getint, bool: parser.getboolean}
 
     return OrderedDict(
         (option.name, type_getter[option.value_type](section_format.name, option.name))
@@ -151,11 +143,10 @@ def parse_configuration(config_filename, config_format):
 
     # Describes a parsed configuration, where each attribute is the name of a configuration file
     # section and each value is a dict of that section's parsed options.
-    Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format))
+    Parsed_config = namedtuple(
+        'Parsed_config', (section_format.name for section_format in config_format)
+    )
 
     return Parsed_config(
-        *(
-            parse_section_options(parser, section_format)
-            for section_format in config_format
-        )
+        *(parse_section_options(parser, section_format) for section_format in config_format)
     )

borgmatic/config/load.py

@@ -0,0 +1,59 @@
+import logging
+import os
+
+import ruamel.yaml
+
+logger = logging.getLogger(__name__)
+
+
+def load_configuration(filename):
+    '''
+    Load the given configuration file and return its contents as a data structure of nested dicts
+    and lists.
+
+    Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
+    if there are too many recursive includes.
+    '''
+    yaml = ruamel.yaml.YAML(typ='safe')
+    yaml.Constructor = Include_constructor
+
+    return yaml.load(open(filename))
+
+
+def include_configuration(loader, filename_node):
+    '''
+    Load the given YAML filename (ignoring the given loader so we can use our own), and return its
+    contents as a data structure of nested dicts and lists.
+    '''
+    return load_configuration(os.path.expanduser(filename_node.value))
+
+
+class Include_constructor(ruamel.yaml.SafeConstructor):
+    '''
+    A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
+    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    '''
+
+    def __init__(self, preserve_quotes=None, loader=None):
+        super(Include_constructor, self).__init__(preserve_quotes, loader)
+        self.add_constructor('!include', include_configuration)
+
+    def flatten_mapping(self, node):
+        '''
+        Support the special case of shallow merging included configuration into an existing mapping
+        using the YAML '<<' merge key. Example syntax:
+
+        ```
+        retention:
+            keep_daily: 1
+            <<: !include common.yaml
+        ```
+        '''
+        representer = ruamel.yaml.representer.SafeRepresenter()
+
+        for index, (key_node, value_node) in enumerate(node.value):
+            if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
+                included_value = representer.represent_data(self.construct_object(value_node))
+                node.value[index] = (key_node, included_value)
+
+        super(Include_constructor, self).flatten_mapping(node)

borgmatic/config/schema.yaml

@@ -11,57 +11,120 @@ map:
             source_directories:
                 required: true
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     List of source directories to backup (required). Globs and tildes are expanded.
                 example:
                     - /home
                     - /etc
                     - /var/log/syslog*
-            one_file_system:
-                type: bool
-                desc: Stay in same file system (do not cross mount points).
-                example: true
-            remote_path:
-                type: scalar
-                desc: Alternate Borg remote executable. Defaults to "borg".
-                example: borg1
             repositories:
                 required: true
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
-                    Paths to local or remote repositories (required). Multiple repositories are
-                    backed up to in sequence.
+                    Paths to local or remote repositories (required). Tildes are expanded. Multiple
+                    repositories are backed up to in sequence. See ssh_command for SSH options like
+                    identity file or port.
                 example:
                     - user@backupserver:sourcehostname.borg
+            one_file_system:
+                type: bool
+                desc: Stay in same file system (do not cross mount points). Defaults to false.
+                example: true
+            numeric_owner:
+                type: bool
+                desc: Only store/extract numeric user and group identifiers. Defaults to false.
+                example: true
+            atime:
+                type: bool
+                desc: Store atime into archive. Defaults to true.
+                example: false
+            ctime:
+                type: bool
+                desc: Store ctime into archive. Defaults to true.
+                example: false
+            birthtime:
+                type: bool
+                desc: Store birthtime (creation date) into archive. Defaults to true.
+                example: false
+            read_special:
+                type: bool
+                desc: |
+                    Use Borg's --read-special flag to allow backup of block and other special
+                    devices. Use with caution, as it will lead to problems if used when
+                    backing up special devices such as /dev/zero. Defaults to false.
+                example: false
+            bsd_flags:
+                type: bool
+                desc: Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true.
+                example: true
+            files_cache:
+                type: str
+                desc: |
+                    Mode in which to operate the files cache. See
+                    https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
+                    details. Defaults to "ctime,size,inode".
+                example: ctime,size,inode
+            local_path:
+                type: str
+                desc: Alternate Borg local executable. Defaults to "borg".
+                example: borg1
+            remote_path:
+                type: str
+                desc: Alternate Borg remote executable. Defaults to "borg".
+                example: borg1
+            patterns:
+                seq:
+                    - type: str
+                desc: |
+                    Any paths matching these patterns are included/excluded from backups. Globs are
+                    expanded. (Tildes are not.) Note that Borg considers this option experimental.
+                    See the output of "borg help patterns" for more details. Quote any value if it
+                    contains leading punctuation, so it parses correctly.
+                example:
+                    - 'R /'
+                    - '- /home/*/.cache'
+                    - '+ /home/susan'
+                    - '- /home/*'
+            patterns_from:
+                seq:
+                    - type: str
+                desc: |
+                    Read include/exclude patterns from one or more separate named files, one pattern
+                    per line. Note that Borg considers this option experimental. See the output of
+                    "borg help patterns" for more details.
+                example:
+                    - /etc/borgmatic/patterns
             exclude_patterns:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
-                    Any paths matching these patterns are excluded from backups. Globs are expanded.
-                    See the output of "borg help patterns" for more details.
+                    Any paths matching these patterns are excluded from backups. Globs and tildes
+                    are expanded. See the output of "borg help patterns" for more details.
                 example:
                     - '*.pyc'
-                    - /home/*/.cache
+                    - ~/*/.cache
                     - /etc/ssl
             exclude_from:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Read exclude patterns from one or more separate named files, one pattern per
-                    line.
+                    line. See the output of "borg help patterns" for more details.
                 example:
                     - /etc/borgmatic/excludes
             exclude_caches:
                 type: bool
                 desc: |
                     Exclude directories that contain a CACHEDIR.TAG file. See
-                    http://www.brynosaurus.com/cachedir/spec.html for details.
+                    http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false.
                 example: true
             exclude_if_present:
-                type: scalar
-                desc: Exclude directories that contain a file with the given filename.
+                type: str
+                desc: |
+                    Exclude directories that contain a file with the given filename. Defaults to not
+                    set.
                 example: .nobackup
     storage:
         desc: |
@@ -70,43 +133,127 @@ map:
             https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for
             details.
         map:
+            encryption_passcommand:
+                type: str
+                desc: |
+                    The standard output of this command is used to unlock the encryption key. Only
+                    use on repositories that were initialized with passcommand/repokey encryption.
+                    Note that if both encryption_passcommand and encryption_passphrase are set,
+                    then encryption_passphrase takes precedence. Defaults to not set.
+                example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
-                type: scalar
+                type: str
                 desc: |
                     Passphrase to unlock the encryption key with. Only use on repositories that were
                     initialized with passphrase/repokey encryption. Quote the value if it contains
                     punctuation, so it parses correctly. And backslash any quote or backslash
-                    literals as well.
+                    literals as well. Defaults to not set.
                 example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
+            checkpoint_interval:
+                type: int
+                desc: |
+                    Number of seconds between each checkpoint during a long-running backup. See
+                    https://borgbackup.readthedocs.io/en/stable/faq.html#if-a-backup-stops-mid-way-does-the-already-backed-up-data-stay-there
+                    for details. Defaults to checkpoints every 1800 seconds (30 minutes).
+                example: 1800
+            chunker_params:
+                type: str
+                desc: |
+                    Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP,
+                    HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html
+                    for details. Defaults to "19,23,21,4095".
+                example: 19,23,21,4095
             compression:
-                type: scalar
+                type: str
                 desc: |
                     Type of compression to use when creating archives. See
                     https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
-                    Defaults to no compression.
+                    Defaults to "lz4".
                 example: lz4
+            remote_rate_limit:
+                type: int
+                desc: Remote network upload rate limit in kiBytes/second. Defaults to unlimited.
+                example: 100
+            ssh_command:
+                type: str
+                desc: |
+                    Command to use instead of "ssh". This can be used to specify ssh options.
+                    Defaults to not set.
+                example: ssh -i /path/to/private/key
+            borg_base_directory:
+                type: str
+                desc: |
+                    Base path used for various Borg directories. Defaults to $HOME, ~$USER, or ~.
+                    See https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for details.
+                example: /path/to/base
+            borg_config_directory:
+                type: str
+                desc: |
+                    Path for Borg configuration files. Defaults to $borg_base_directory/.config/borg
+                example: /path/to/base/config
+            borg_cache_directory:
+                type: str
+                desc: |
+                    Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg
+                example: /path/to/base/cache
+            borg_security_directory:
+                type: str
+                desc: |
+                    Path for Borg security and encryption nonce files. Defaults to $borg_base_directory/.config/borg/security
+                example: /path/to/base/config/security
+            borg_keys_directory:
+                type: str
+                desc: |
+                    Path for Borg encryption key files. Defaults to $borg_base_directory/.config/borg/keys
+                example: /path/to/base/config/keys
             umask:
                 type: scalar
-                desc: Umask to be used for borg create.
+                desc: Umask to be used for borg create. Defaults to 0077.
                 example: 0077
+            lock_wait:
+                type: int
+                desc: Maximum seconds to wait for acquiring a repository/cache lock. Defaults to 1.
+                example: 5
             archive_name_format:
-                type: scalar
+                type: str
                 desc: |
                     Name of the archive. Borg placeholders can be used. See the output of
-                    "borg help placeholders" for details. Default is
+                    "borg help placeholders" for details. Defaults to
                     "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must
                     also specify a prefix in the retention section to avoid accidental pruning of
-                    archives with a different archive name format.
+                    archives with a different archive name format. And you should also specify a
+                    prefix in the consistency section as well.
                 example: "{hostname}-documents-{now}"
+            relocated_repo_access_is_ok:
+                type: bool
+                desc: Bypass Borg error about a repository that has been moved. Defaults to false.
+                example: true
+            unknown_unencrypted_repo_access_is_ok:
+                type: bool
+                desc: |
+                    Bypass Borg error about a previously unknown unencrypted repository. Defaults to
+                    false.
+                example: true
     retention:
         desc: |
             Retention policy for how many backups to keep in each category. See
             https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
+            At least one of the "keep" options is required for pruning to work. See
+            https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/
+            if you'd like to skip pruning entirely.
         map:
             keep_within:
-                type: scalar
+                type: str
                 desc: Keep all archives within this time interval.
                 example: 3H
+            keep_secondly:
+                type: int
+                desc: Number of secondly archives to keep.
+                example: 60
+            keep_minutely:
+                type: int
+                desc: Number of minutely archives to keep.
+                example: 60
             keep_hourly:
                 type: int
                 desc: Number of hourly archives to keep.
@@ -128,11 +275,11 @@ map:
                 desc: Number of yearly archives to keep.
                 example: 1
             prefix:
-                type: scalar
+                type: str
                 desc: |
                     When pruning, only consider archive names starting with this prefix.
                     Borg placeholders can be used. See the output of "borg help placeholders" for
-                    details. Default is "{hostname}-".
+                    details. Defaults to "{hostname}-". Use an empty value to disable the default.
                 example: sourcehostname
     consistency:
         desc: |
@@ -143,44 +290,163 @@ map:
             checks:
                 seq:
                     - type: str
-                      enum: ['repository', 'archives', 'extract', 'disabled']
+                      enum: ['repository', 'archives', 'data', 'extract', 'disabled']
                       unique: true
                 desc: |
-                    List of one or more consistency checks to run: "repository", "archives", and/or
-                    "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
-                    all consistency checks. "repository" checks the consistency of the repository,
-                    "archive" checks all of the archives, and "extract" does an extraction dry-run
-                    of just the most recent archive.
+                    List of one or more consistency checks to run: "repository", "archives", "data",
+                    and/or "extract". Defaults to "repository" and "archives". Set to "disabled" to
+                    disable all consistency checks. "repository" checks the consistency of the
+                    repository, "archives" checks all of the archives, "data" verifies the integrity
+                    of the data within the archives, and "extract" does an extraction dry-run of the
+                    most recent archive. Note that "data" implies "archives".
                 example:
                     - repository
                     - archives
+            check_repositories:
+                seq:
+                    - type: str
+                desc: |
+                    Paths to a subset of the repositories in the location section on which to run
+                    consistency checks. Handy in case some of your repositories are very large, and
+                    so running consistency checks on them would take too long. Defaults to running
+                    consistency checks on all repositories configured in the location section.
+                example:
+                    - user@backupserver:sourcehostname.borg
             check_last:
                 type: int
                 desc: Restrict the number of checked archives to the last n. Applies only to the
-                      "archives" check.
+                      "archives" check. Defaults to checking all archives.
                 example: 3
+            prefix:
+                type: str
+                desc: |
+                    When performing the "archives" check, only consider archive names starting with
+                    this prefix. Borg placeholders can be used. See the output of
+                    "borg help placeholders" for details. Defaults to "{hostname}-". Use an empty
+                    value to disable the default.
+                example: sourcehostname
+    output:
+        desc: |
+            Options for customizing borgmatic's own output and logging.
+        map:
+            color:
+                type: bool
+                desc: |
+                    Apply color to console output. Can be overridden with --no-color command-line
+                    flag. Defaults to true.
+                example: false
     hooks:
         desc: |
-            Shell commands or scripts to execute before and after a backup or if an error has occurred.
-            IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
-            Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
-            prevent potential shell injection or privilege escalation.
+            Shell commands, scripts, or integrations to execute at various points during a borgmatic
+            run. IMPORTANT: All provided commands and scripts are executed with user permissions of
+            borgmatic. Do not forget to set secure permissions on this configuration file (chmod
+            0600) as well as on any script called from a hook (chmod 0700) to prevent potential
+            shell injection or privilege escalation.
         map:
             before_backup:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute before creating a backup.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute before creating a
+                    backup, run once per configuration file.
                 example:
-                    - echo "`date` - Starting a backup job."
+                    - echo "Starting a backup."
             after_backup:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute after creating a backup.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute after creating a
+                    backup, run once per configuration file.
                 example:
-                    - echo "`date` - Backup created."
+                    - echo "Created a backup."
             on_error:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute in case an exception has occurred.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute when an exception
+                    occurs during a backup or when running a before_backup or after_backup hook.
                 example:
-                    - echo "`date` - Error while creating a backup."
+                    - echo "Error while creating a backup or running a backup hook."
+            postgresql_databases:
+                seq:
+                    - map:
+                        name:
+                            required: true
+                            type: str
+                            desc: |
+                                Database name (required if using this hook). Or "all" to dump all
+                                databases on the host.
+                            example: users
+                        hostname:
+                            type: str
+                            desc: |
+                                Database hostname to connect to. Defaults to connecting via local
+                                Unix socket.
+                            example: database.example.org
+                        port:
+                            type: int
+                            desc: Port to connect to. Defaults to 5432.
+                            example: 5433
+                        username:
+                            type: str
+                            desc: |
+                                Username with which to connect to the database. Defaults to the
+                                username of the current user. You probably want to specify the
+                                "postgres" superuser here when the database name is "all".
+                            example: dbuser
+                        password:
+                            type: str
+                            desc: |
+                                Password with which to connect to the database. Omitting a password
+                                will only work if PostgreSQL is configured to trust the configured
+                                username without a password, or you create a ~/.pgpass file.
+                            example: trustsome1
+                        format:
+                            type: str
+                            enum: ['plain', 'custom', 'directory', 'tar']
+                            desc: |
+                                Database dump output format. One of "plain", "custom", "directory",
+                                or "tar". Defaults to "custom" (unlike raw pg_dump). See
+                                https://www.postgresql.org/docs/current/app-pgdump.html for details.
+                                Note that format is ignored when the database name is "all".
+                            example: directory
+                        options:
+                            type: str
+                            desc: |
+                                Additional pg_dump/pg_dumpall options to pass directly to the dump
+                                command, without performing any validation on them. See
+                                https://www.postgresql.org/docs/current/app-pgdump.html for details.
+                            example: --role=someone
+                desc: |
+                    List of one or more PostgreSQL databases to dump before creating a backup,
+                    run once per configuration file. The database dumps are added to your source
+                    directories at runtime, backed up, and then removed afterwards. Requires
+                    pg_dump/pg_dumpall/pg_restore commands. See
+                    https://www.postgresql.org/docs/current/app-pgdump.html for details.
+            healthchecks:
+                type: str
+                desc: |
+                    Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
+                    Create an account at https://healthchecks.io if you'd like to use this service.
+                example:
+                    https://hc-ping.com/your-uuid-here
+            before_everything:
+                seq:
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute before running all
+                    actions (if one of them is "create"), run once before all configuration files.
+                example:
+                    - echo "Starting actions."
+            after_everything:
+                seq:
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute after running all
+                    actions (if one of them is "create"), run once after all configuration files.
+                example:
+                    - echo "Completed actions."
+            umask:
+                type: scalar
+                desc: Umask used when executing hooks. Defaults to the umask that borgmatic is run with.
+                example: 0077

borgmatic/config/validate.py

@@ -1,11 +1,11 @@
 import logging
-import sys
-import warnings
 
 import pkg_resources
 import pykwalify.core
 import pykwalify.errors
-from ruamel import yaml
+import ruamel.yaml
+
+from borgmatic.config import load
 
 
 def schema_filename():
@@ -21,6 +21,7 @@ class Validation_error(ValueError):
     A collection of error message strings generated when attempting to validate a particular
     configurartion file.
     '''
+
     def __init__(self, config_filename, error_messages):
         self.config_filename = config_filename
         self.error_messages = error_messages
@@ -45,11 +46,40 @@ def apply_logical_validation(config_filename, parsed_configuration):
 
     if archive_name_format and not prefix:
         raise Validation_error(
-            config_filename, (
-                'If you provide an archive_name_format, you must also specify a retention prefix.',
-            )
+            config_filename,
+            ('If you provide an archive_name_format, you must also specify a retention prefix.',),
         )
 
+    location_repositories = parsed_configuration.get('location', {}).get('repositories')
+    check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', [])
+    for repository in check_repositories:
+        if repository not in location_repositories:
+            raise Validation_error(
+                config_filename,
+                (
+                    'Unknown repository in the consistency section\'s check_repositories: {}'.format(
+                        repository
+                    ),
+                ),
+            )
+
+
+def remove_examples(schema):
+    '''
+    pykwalify gets angry if the example field is not a string. So rather than bend to its will,
+    remove all examples from the given schema before passing the schema to pykwalify.
+    '''
+    if 'map' in schema:
+        for item_name, item_schema in schema['map'].items():
+            item_schema.pop('example', None)
+            remove_examples(item_schema)
+    elif 'seq' in schema:
+        for item_schema in schema['seq']:
+            item_schema.pop('example', None)
+            remove_examples(item_schema)
+
+    return schema
+
 
 def parse_configuration(config_filename, schema_filename):
     '''
@@ -66,18 +96,12 @@ def parse_configuration(config_filename, schema_filename):
     logging.getLogger('pykwalify').setLevel(logging.ERROR)
 
     try:
-        config = yaml.round_trip_load(open(config_filename))
-        schema = yaml.round_trip_load(open(schema_filename))
-    except yaml.error.YAMLError as error:
+        config = load.load_configuration(config_filename)
+        schema = load.load_configuration(schema_filename)
+    except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
-    # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
-    # simply remove all examples before passing the schema to pykwalify.
-    for section_name, section_schema in schema['map'].items():
-        for field_name, field_schema in section_schema['map'].items():
-            field_schema.pop('example', None)
-
-    validator = pykwalify.core.Core(source_data=config, schema_data=schema)
+    validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
     parsed_result = validator.validate(raise_exception=False)
 
     if validator.validation_errors:
@@ -86,3 +110,46 @@ def parse_configuration(config_filename, schema_filename):
     apply_logical_validation(config_filename, parsed_result)
 
     return parsed_result
+
+
+def guard_configuration_contains_repository(repository, configurations):
+    '''
+    Given a repository path and a dict mapping from config filename to corresponding parsed config
+    dict, ensure that the repository is declared exactly once in all of the configurations.
+
+    If no repository is given, then error if there are multiple configured repositories.
+
+    Raise ValueError if the repository is not found in a configuration, or is declared multiple
+    times.
+    '''
+    if not repository:
+        count = len(
+            tuple(
+                config_repository
+                for config in configurations.values()
+                for config_repository in config['location']['repositories']
+            )
+        )
+
+        if count > 1:
+            raise ValueError(
+                'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
+                    repository
+                )
+            )
+
+        return
+
+    count = len(
+        tuple(
+            config_repository
+            for config in configurations.values()
+            for config_repository in config['location']['repositories']
+            if repository == config_repository
+        )
+    )
+
+    if count == 0:
+        raise ValueError('Repository {} not found in configuration files'.format(repository))
+    if count > 1:
+        raise ValueError('Repository {} found in multiple configuration files'.format(repository))

borgmatic/execute.py

@@ -0,0 +1,94 @@
+import logging
+import os
+import subprocess
+
+logger = logging.getLogger(__name__)
+
+
+ERROR_OUTPUT_MAX_LINE_COUNT = 25
+BORG_ERROR_EXIT_CODE = 2
+
+
+def borg_command(full_command):
+    '''
+    Return True if this is a Borg command, or False if it's some other command.
+    '''
+    return 'borg' in full_command[0]
+
+
+def execute_and_log_output(full_command, output_log_level, shell, environment):
+    last_lines = []
+    process = subprocess.Popen(
+        full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, env=environment
+    )
+
+    while process.poll() is None:
+        line = process.stdout.readline().rstrip().decode()
+        if not line:
+            continue
+
+        # Keep the last few lines of output in case the command errors, and we need the output for
+        # the exception below.
+        last_lines.append(line)
+        if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.pop(0)
+
+        logger.log(output_log_level, line)
+
+    remaining_output = process.stdout.read().rstrip().decode()
+    if remaining_output:  # pragma: no cover
+        logger.log(output_log_level, remaining_output)
+
+    exit_code = process.poll()
+
+    # If we're running something other than Borg, treat all non-zero exit codes as errors.
+    if borg_command(full_command):
+        error = bool(exit_code >= BORG_ERROR_EXIT_CODE)
+    else:
+        error = bool(exit_code != 0)
+
+    if error:
+        # If an error occurs, include its output in the raised exception so that we don't
+        # inadvertently hide error output.
+        if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.insert(0, '...')
+
+        raise subprocess.CalledProcessError(
+            exit_code, ' '.join(full_command), '\n'.join(last_lines)
+        )
+
+
+def execute_command(
+    full_command, output_log_level=logging.INFO, shell=False, extra_environment=None
+):
+    '''
+    Execute the given command (a sequence of command/argument strings) and log its output at the
+    given log level. If output log level is None, instead capture and return the output. If
+    shell is True, execute the command within a shell. If an extra environment dict is given, then
+    use it to augment the current environment, and pass the result into the command.
+
+    Raise subprocesses.CalledProcessError if an error occurs while running the command.
+    '''
+    logger.debug(' '.join(full_command))
+    environment = {**os.environ, **extra_environment} if extra_environment else None
+
+    if output_log_level is None:
+        output = subprocess.check_output(full_command, shell=shell, env=environment)
+        return output.decode() if output is not None else None
+    else:
+        execute_and_log_output(full_command, output_log_level, shell=shell, environment=environment)
+
+
+def execute_command_without_capture(full_command):
+    '''
+    Execute the given command (a sequence of command/argument strings), but don't capture or log its
+    output in any way. This is necessary for commands that monkey with the terminal (e.g. progress
+    display) or provide interactive prompts.
+    '''
+    logger.debug(' '.join(full_command))
+
+    try:
+        subprocess.check_call(full_command)
+    except subprocess.CalledProcessError as error:
+        if error.returncode >= BORG_ERROR_EXIT_CODE:
+            raise

borgmatic/hooks/command.py

@@ -0,0 +1,71 @@
+import logging
+import os
+
+from borgmatic import execute
+
+logger = logging.getLogger(__name__)
+
+
+def interpolate_context(command, context):
+    '''
+    Given a single hook command and a dict of context names/values, interpolate the values by
+    "{name}" into the command and return the result.
+    '''
+    for name, value in context.items():
+        command = command.replace('{%s}' % name, str(value))
+
+    return command
+
+
+def execute_hook(commands, umask, config_filename, description, dry_run, **context):
+    '''
+    Given a list of hook commands to execute, a umask to execute with (or None), a config filename,
+    a hook description, and whether this is a dry run, run the given commands. Or, don't run them
+    if this is a dry run.
+
+    The context contains optional values interpolated by name into the hook commands. Currently,
+    this only applies to the on_error hook.
+
+    Raise ValueError if the umask cannot be parsed.
+    Raise subprocesses.CalledProcessError if an error occurs in a hook.
+    '''
+    if not commands:
+        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
+        return
+
+    dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
+
+    context['configuration_filename'] = config_filename
+    commands = [interpolate_context(command, context) for command in commands]
+
+    if len(commands) == 1:
+        logger.info(
+            '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label)
+        )
+    else:
+        logger.info(
+            '{}: Running {} commands for {} hook{}'.format(
+                config_filename, len(commands), description, dry_run_label
+            )
+        )
+
+    if umask:
+        parsed_umask = int(str(umask), 8)
+        logger.debug('{}: Set hook umask to {}'.format(config_filename, oct(parsed_umask)))
+        original_umask = os.umask(parsed_umask)
+    else:
+        original_umask = None
+
+    try:
+        for command in commands:
+            if not dry_run:
+                execute.execute_command(
+                    [command],
+                    output_log_level=logging.ERROR
+                    if description == 'on-error'
+                    else logging.WARNING,
+                    shell=True,
+                )
+    finally:
+        if original_umask:
+            os.umask(original_umask)

borgmatic/hooks/healthchecks.py

@@ -0,0 +1,36 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
+    '''
+    Ping the given healthchecks.io URL or UUID, appending the append string if any. Use the given
+    configuration filename in any log entries. If this is a dry run, then don't actually ping
+    anything.
+    '''
+    if not ping_url_or_uuid:
+        logger.debug('{}: No healthchecks hook set'.format(config_filename))
+        return
+
+    ping_url = (
+        ping_url_or_uuid
+        if ping_url_or_uuid.startswith('http')
+        else 'https://hc-ping.com/{}'.format(ping_url_or_uuid)
+    )
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+
+    if append:
+        ping_url = '{}/{}'.format(ping_url, append)
+
+    logger.info(
+        '{}: Pinging healthchecks.io{}{}'.format(
+            config_filename, ' ' + append if append else '', dry_run_label
+        )
+    )
+    logger.debug('{}: Using healthchecks.io ping URL {}'.format(config_filename, ping_url))
+
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    requests.get(ping_url)

borgmatic/hooks/postgresql.py

@@ -0,0 +1,88 @@
+import logging
+import os
+
+from borgmatic.execute import execute_command
+
+DUMP_PATH = '~/.borgmatic/postgresql_databases'
+logger = logging.getLogger(__name__)
+
+
+def dump_databases(databases, config_filename, dry_run):
+    '''
+    Dump the given PostgreSQL databases to disk. The databases are supplied as a sequence of dicts,
+    one dict describing each database as per the configuration schema. Use the given configuration
+    filename in any log entries. If this is a dry run, then don't actually dump anything.
+    '''
+    if not databases:
+        logger.debug('{}: No PostgreSQL databases configured'.format(config_filename))
+        return
+
+    dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+
+    logger.info('{}: Dumping PostgreSQL databases{}'.format(config_filename, dry_run_label))
+
+    for database in databases:
+        if os.path.sep in database['name']:
+            raise ValueError('Invalid database name {}'.format(database['name']))
+
+        dump_path = os.path.join(
+            os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost')
+        )
+        name = database['name']
+        all_databases = bool(name == 'all')
+        command = (
+            ('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean')
+            + ('--file', os.path.join(dump_path, name))
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--username', database['username']) if 'username' in database else ())
+            + (() if all_databases else ('--format', database.get('format', 'custom')))
+            + (tuple(database['options'].split(' ')) if 'options' in database else ())
+            + (() if all_databases else (name,))
+        )
+        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+
+        logger.debug(
+            '{}: Dumping PostgreSQL database {}{}'.format(config_filename, name, dry_run_label)
+        )
+        if not dry_run:
+            os.makedirs(dump_path, mode=0o700, exist_ok=True)
+            execute_command(command, extra_environment=extra_environment)
+
+
+def remove_database_dumps(databases, config_filename, dry_run):
+    '''
+    Remove the database dumps for the given databases. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the given
+    configuration filename in any log entries. If this is a dry run, then don't actually remove
+    anything.
+    '''
+    if not databases:
+        logger.debug('{}: No PostgreSQL databases configured'.format(config_filename))
+        return
+
+    dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
+
+    logger.info('{}: Removing PostgreSQL database dumps{}'.format(config_filename, dry_run_label))
+
+    for database in databases:
+        if os.path.sep in database['name']:
+            raise ValueError('Invalid database name {}'.format(database['name']))
+
+        name = database['name']
+        dump_path = os.path.join(
+            os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost')
+        )
+        dump_filename = os.path.join(dump_path, name)
+
+        logger.debug(
+            '{}: Remove PostgreSQL database dump {} from {}{}'.format(
+                config_filename, name, dump_filename, dry_run_label
+            )
+        )
+        if dry_run:
+            continue
+
+        os.remove(dump_filename)
+        if len(os.listdir(dump_path)) == 0:
+            os.rmdir(dump_path)

borgmatic/logger.py

@@ -0,0 +1,101 @@
+import logging
+import os
+import sys
+
+import colorama
+
+
+def to_bool(arg):
+    '''
+    Return a boolean value based on `arg`.
+    '''
+    if arg is None or isinstance(arg, bool):
+        return arg
+
+    if isinstance(arg, str):
+        arg = arg.lower()
+
+    if arg in ('yes', 'on', '1', 'true', 1):
+        return True
+
+    return False
+
+
+def interactive_console():
+    '''
+    Return whether the current console is "interactive". Meaning: Capable of
+    user input and not just something like a cron job.
+    '''
+    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+
+
+def should_do_markup(no_color, configs):
+    '''
+    Given the value of the command-line no-color argument, and a dict of configuration filename to
+    corresponding parsed configuration, determine if we should enable colorama marking up.
+    '''
+    if no_color:
+        return False
+
+    if any(config.get('output', {}).get('color') is False for config in configs.values()):
+        return False
+
+    py_colors = os.environ.get('PY_COLORS', None)
+
+    if py_colors is not None:
+        return to_bool(py_colors)
+
+    return interactive_console()
+
+
+LOG_LEVEL_TO_COLOR = {
+    logging.CRITICAL: colorama.Fore.RED,
+    logging.ERROR: colorama.Fore.RED,
+    logging.WARN: colorama.Fore.YELLOW,
+    logging.INFO: colorama.Fore.GREEN,
+    logging.DEBUG: colorama.Fore.CYAN,
+}
+
+
+class Console_color_formatter(logging.Formatter):
+    def format(self, record):
+        color = LOG_LEVEL_TO_COLOR.get(record.levelno)
+        return color_text(color, record.msg)
+
+
+def color_text(color, message):
+    '''
+    Give colored text.
+    '''
+    if not color:
+        return message
+
+    return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL)
+
+
+def configure_logging(console_log_level, syslog_log_level=None):
+    '''
+    Configure logging to go to both the console and syslog. Use the given log levels, respectively.
+    '''
+    if syslog_log_level is None:
+        syslog_log_level = console_log_level
+
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(Console_color_formatter())
+    console_handler.setLevel(console_log_level)
+
+    syslog_path = None
+    if os.path.exists('/dev/log'):
+        syslog_path = '/dev/log'
+    elif os.path.exists('/var/run/syslog'):
+        syslog_path = '/var/run/syslog'
+
+    if syslog_path and not interactive_console():
+        syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
+        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
+        syslog_handler.setLevel(syslog_log_level)
+        handlers = (console_handler, syslog_handler)
+    else:
+        handlers = (console_handler,)
+
+    logging.basicConfig(level=min(console_log_level, syslog_log_level), handlers=handlers)

borgmatic/signals.py

@@ -0,0 +1,18 @@
+import os
+import signal
+
+
+def _handle_signal(signal_number, frame):  # pragma: no cover
+    '''
+    Send the signal to all processes in borgmatic's process group, which includes child process.
+    '''
+    os.killpg(os.getpgrp(), signal_number)
+
+
+def configure_signals():  # pragma: no cover
+    '''
+    Configure borgmatic's signal handlers to pass relevant signals through to any child processes
+    like Borg. Note that SIGINT gets passed through even without these changes.
+    '''
+    for signal_number in (signal.SIGHUP, signal.SIGTERM, signal.SIGUSR1, signal.SIGUSR2):
+        signal.signal(signal_number, _handle_signal)

borgmatic/tests/integration/commands/test_borgmatic.py

@@ -1,66 +0,0 @@
-import os
-
-from flexmock import flexmock
-import pytest
-
-from borgmatic.commands import borgmatic as module
-
-
-def test_parse_arguments_with_no_arguments_uses_defaults():
-    parser = module.parse_arguments()
-
-    assert parser.config_paths == module.collect.DEFAULT_CONFIG_PATHS
-    assert parser.excludes_filename == None
-    assert parser.verbosity is None
-
-
-def test_parse_arguments_with_path_arguments_overrides_defaults():
-    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
-
-    assert parser.config_paths == ['myconfig']
-    assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity is None
-
-
-def test_parse_arguments_with_multiple_config_paths_parses_as_list():
-    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
-
-    assert parser.config_paths == ['myconfig', 'otherconfig']
-    assert parser.verbosity is None
-
-
-def test_parse_arguments_with_verbosity_flag_overrides_default():
-    parser = module.parse_arguments('--verbosity', '1')
-
-    assert parser.config_paths == module.collect.DEFAULT_CONFIG_PATHS
-    assert parser.excludes_filename == None
-    assert parser.verbosity == 1
-
-
-def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
-    parser = module.parse_arguments()
-
-    assert parser.prune is True
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
-    parser = module.parse_arguments('--prune')
-
-    assert parser.prune is True
-    assert parser.create is False
-    assert parser.check is False
-
-
-def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
-    parser = module.parse_arguments('--create', '--check')
-
-    assert parser.prune is False
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_invalid_arguments_exits():
-    with pytest.raises(SystemExit):
-        module.parse_arguments('--posix-me-harder')

borgmatic/tests/integration/config/test_generate.py

@@ -1,65 +0,0 @@
-from io import StringIO
-import os
-import sys
-
-from flexmock import flexmock
-import pytest
-
-from borgmatic.config import generate as module
-
-
-def test_insert_newline_before_comment_does_not_raise():
-    field_name = 'foo'
-    config = module.yaml.comments.CommentedMap([(field_name, 33)])
-    config.yaml_set_comment_before_after_key(key=field_name, before='Comment',)
-
-    module._insert_newline_before_comment(config, field_name)
-
-
-def test_write_configuration_does_not_raise():
-    flexmock(os.path).should_receive('exists').and_return(False)
-    flexmock(os).should_receive('makedirs')
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').and_return(StringIO())
-    flexmock(os).should_receive('chmod')
-
-    module.write_configuration('config.yaml', {})
-
-
-def test_write_configuration_with_already_existing_file_raises():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
-    with pytest.raises(FileExistsError):
-        module.write_configuration('config.yaml', {})
-
-
-def test_write_configuration_with_already_existing_directory_does_not_raise():
-    flexmock(os.path).should_receive('exists').and_return(False)
-    flexmock(os).should_receive('makedirs').and_raise(FileExistsError)
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').and_return(StringIO())
-    flexmock(os).should_receive('chmod')
-
-    module.write_configuration('config.yaml', {})
-
-
-def test_add_comments_to_configuration_does_not_raise():
-    # Ensure that it can deal with fields both in the schema and missing from the schema.
-    config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
-    schema = {
-        'map': {
-            'foo': {'desc': 'Foo'},
-            'bar': {'desc': 'Bar'},
-        }
-    }
-
-    module.add_comments_to_configuration(config, schema)
-
-
-def test_generate_sample_configuration_does_not_raise():
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').with_args('schema.yaml').and_return('')
-    flexmock(module).should_receive('write_configuration')
-    flexmock(module).should_receive('_schema_to_sample_configuration')
-
-    module.generate_sample_configuration('config.yaml', 'schema.yaml')

borgmatic/tests/integration/test_version.py

@@ -1,8 +0,0 @@
-import subprocess
-
-
-def test_setup_version_matches_news_version():
-    setup_version = subprocess.check_output(('python', 'setup.py', '--version')).decode('ascii')
-    news_version = open('NEWS').readline()
-
-    assert setup_version == news_version

borgmatic/tests/unit/borg/test_check.py

@@ -1,185 +0,0 @@
-from subprocess import STDOUT
-import sys
-
-from flexmock import flexmock
-import pytest
-
-from borgmatic.borg import check as module
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-
-
-def insert_subprocess_never():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').never()
-
-
-def test_parse_checks_returns_them_as_tuple():
-    checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']})
-
-    assert checks == ('foo', 'bar')
-
-
-def test_parse_checks_with_missing_value_returns_defaults():
-    checks = module._parse_checks({})
-
-    assert checks == module.DEFAULT_CHECKS
-
-
-def test_parse_checks_with_blank_value_returns_defaults():
-    checks = module._parse_checks({'checks': []})
-
-    assert checks == module.DEFAULT_CHECKS
-
-
-def test_parse_checks_with_disabled_returns_no_checks():
-    checks = module._parse_checks({'checks': ['disabled']})
-
-    assert checks == ()
-
-
-def test_make_check_flags_with_checks_returns_flags():
-    flags = module._make_check_flags(('repository',))
-
-    assert flags == ('--repository-only',)
-
-
-def test_make_check_flags_with_extract_check_does_not_make_extract_flag():
-    flags = module._make_check_flags(('extract',))
-
-    assert flags == ()
-
-
-def test_make_check_flags_with_default_checks_returns_no_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS)
-
-    assert flags == ()
-
-
-def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
-    flags = module._make_check_flags(('repository',), check_last=3)
-
-    assert flags == ('--repository-only', '--last', '3')
-
-
-def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
-
-    assert flags == ('--last', '3')
-
-
-@pytest.mark.parametrize(
-    'checks',
-    (
-        ('repository',),
-        ('archives',),
-        ('repository', 'archives'),
-        ('repository', 'archives', 'other'),
-    ),
-)
-def test_check_archives_should_call_borg_with_parameters(checks):
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo'),
-        stdout=stdout, stderr=STDOUT,
-    )
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_with_extract_check_should_call_extract_only():
-    checks = ('extract',)
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').never()
-    flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
-    insert_subprocess_never()
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
-    checks = ('repository',)
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--info'),
-        stdout=None, stderr=STDOUT,
-    )
-
-    module.check_archives(
-        verbosity=VERBOSITY_SOME,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    checks = ('repository',)
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--debug'),
-        stdout=None, stderr=STDOUT,
-    )
-
-    module.check_archives(
-        verbosity=VERBOSITY_LOTS,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_without_any_checks_should_bail():
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(())
-    insert_subprocess_never()
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
-    checks = ('repository',)
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--remote-path', 'borg1'),
-        stdout=stdout, stderr=STDOUT,
-    )
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-        remote_path='borg1',
-    )

borgmatic/tests/unit/borg/test_create.py

@@ -1,383 +0,0 @@
-import os
-
-from flexmock import flexmock
-
-from borgmatic.borg import create as module
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-def test_initialize_with_passphrase_should_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize({'encryption_passphrase': 'pass'})
-        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
-    finally:
-        os.environ = orig_environ
-
-
-def test_initialize_without_passphrase_should_not_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize({})
-        assert os.environ.get('BORG_PASSPHRASE') == None
-    finally:
-        os.environ = orig_environ
-
-
-def test_expand_directory_with_basic_path_passes_it_through():
-    flexmock(module.os.path).should_receive('expanduser').and_return('foo')
-    flexmock(module.glob).should_receive('glob').and_return([])
-
-    paths = module._expand_directory('foo')
-
-    assert paths == ['foo']
-
-
-def test_expand_directory_with_glob_expands():
-    flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
-    flexmock(module.glob).should_receive('glob').and_return(['foo', 'food'])
-
-    paths = module._expand_directory('foo*')
-
-    assert paths == ['foo', 'food']
-
-
-def test_write_exclude_file_does_not_raise():
-    temporary_file = flexmock(
-        name='filename',
-        write=lambda mode: None,
-        flush=lambda: None,
-    )
-    flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
-
-    module._write_exclude_file(['exclude'])
-
-
-def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
-    module._write_exclude_file([])
-
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-
-
-def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_patterns': ['*.pyc', '/var']},
-        exclude_patterns_filename='/tmp/excludes',
-    )
-
-    assert exclude_flags == ('--exclude-from', '/tmp/excludes')
-
-
-def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_from': ['excludes', 'other']},
-    )
-
-    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', 'other')
-
-
-def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_from': ['excludes']},
-        exclude_patterns_filename='/tmp/excludes',
-    )
-
-    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
-
-
-def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_from': None},
-    )
-
-    assert exclude_flags == ()
-
-
-def test_make_exclude_flags_includes_exclude_caches_when_true_in_config():
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_caches': True},
-    )
-
-    assert exclude_flags == ('--exclude-caches',)
-
-
-def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config():
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_caches': False},
-    )
-
-    assert exclude_flags == ()
-
-
-def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_if_present': 'exclude_me'},
-    )
-
-    assert exclude_flags == ('--exclude-if-present', 'exclude_me')
-
-
-def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
-    exclude_flags = module._make_exclude_flags(location_config={})
-
-    assert exclude_flags == ()
-
-
-DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
-CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
-
-
-def test_create_archive_calls_borg_with_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND)
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
-    exclude_flags = ('--exclude-from', 'excludes')
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes'))
-    flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
-    insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': ['exclude'],
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
-
-    module.create_archive(
-        verbosity=VERBOSITY_SOME,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
-
-    module.create_archive(
-        verbosity=VERBOSITY_LOTS,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_compression_calls_borg_with_compression_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
-