.drone.yml

@@ -0,0 +1,54 @@
+kind: pipeline
+name: python-3-8-alpine-3-13
+
+services:
+  - name: postgresql
+    image: postgres:13.1-alpine
+    environment:
+      POSTGRES_PASSWORD: test
+      POSTGRES_DB: test
+  - name: mysql
+    image: mariadb:10.5
+    environment:
+      MYSQL_ROOT_PASSWORD: test
+      MYSQL_DATABASE: test
+  - name: mongodb
+    image: mongo:5.0.5
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: test
+
+clone:
+  skip_verify: true
+
+steps:
+- name: build
+  image: alpine:3.13
+  pull: always
+  commands:
+    - scripts/run-full-tests
+---
+kind: pipeline
+name: documentation
+
+clone:
+  skip_verify: true
+
+steps:
+- name: build
+  image: plugins/docker
+  settings:
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+    repo: witten/borgmatic-docs
+    dockerfile: docs/Dockerfile
+
+trigger:
+  repo:
+    - borgmatic-collective/borgmatic
+  branch:
+    - master
+  event:
+    - push

.eleventy.js

@@ -1,5 +1,4 @@
 const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
-const codeClipboard = require("eleventy-plugin-code-clipboard");
 const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language");
 const navigationPlugin = require("@11ty/eleventy-navigation");
 
@@ -7,7 +6,6 @@ module.exports = function(eleventyConfig) {
     eleventyConfig.addPlugin(pluginSyntaxHighlight);
     eleventyConfig.addPlugin(inclusiveLangPlugin);
     eleventyConfig.addPlugin(navigationPlugin);
-    eleventyConfig.addPlugin(codeClipboard);
 
     let markdownIt = require("markdown-it");
     let markdownItAnchor = require("markdown-it-anchor");
@@ -25,7 +23,8 @@ module.exports = function(eleventyConfig) {
         }
     };
     let markdownItAnchorOptions = {
-        permalink: markdownItAnchor.permalink.headerLink()
+        permalink: true,
+        permalinkClass: "direct-link"
     };
 
     eleventyConfig.setLibrary(
@@ -33,7 +32,6 @@ module.exports = function(eleventyConfig) {
         markdownIt(markdownItOptions)
             .use(markdownItAnchor, markdownItAnchorOptions)
             .use(markdownItReplaceLink)
-            .use(codeClipboard.markdownItCopyButton)
     );
 
     eleventyConfig.addPassthroughCopy({"docs/static": "static"});

.gitea/issue_template.md

@@ -0,0 +1,35 @@
+#### What I'm trying to do and why
+
+#### Steps to reproduce (if a bug)
+
+Include (sanitized) borgmatic configuration files if applicable.
+
+#### Actual behavior (if a bug)
+
+Include (sanitized) `--verbosity 2` output if applicable.
+
+#### Expected behavior (if a bug)
+
+#### Other notes / implementation ideas
+
+#### Environment
+
+**borgmatic version:** [version here]
+
+Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
+
+**borgmatic installation method:** [e.g., Debian package, Docker container, etc.]
+
+**Borg version:** [version here]
+
+Use `sudo borg --version`
+
+**Python version:** [version here]
+
+Use `python3 --version`
+
+**Database version (if applicable):** [version here]
+
+Use `psql --version` or `mysql --version` on client and server.
+
+**operating system and version:** [OS here]

.gitea/issue_template/bug_template.yaml

@@ -1,77 +0,0 @@
-name: "Bug or question/support"
-about: "For filing a bug or getting support"
-body:
-  - type: textarea
-    id: problem
-    attributes:
-      label: What I'm trying to do and why
-    validations:
-      required: true
-  - type: textarea
-    id: repro_steps
-    attributes:
-      label: Steps to reproduce
-      description: Include (sanitized) borgmatic configuration files if applicable.
-    validations:
-      required: false
-  - type: textarea
-    id: actual_behavior
-    attributes:
-      label: Actual behavior
-      description: Include (sanitized) `--verbosity 2` output if applicable.
-    validations:
-      required: false
-  - type: textarea
-    id: expected_behavior
-    attributes:
-      label: Expected behavior
-    validations:
-      required: false
-  - type: textarea
-    id: notes
-    attributes:
-      label: Other notes / implementation ideas
-    validations:
-      required: false
-  - type: input
-    id: borgmatic_version
-    attributes:
-      label: borgmatic version
-      description: Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
-    validations:
-      required: false
-  - type: input
-    id: borgmatic_install_method
-    attributes:
-      label: borgmatic installation method
-      description: e.g., pip install, Debian package, container, etc.
-    validations:
-      required: false
-  - type: input
-    id: borg_version
-    attributes:
-      label: Borg version
-      description: Use `sudo borg --version`
-    validations:
-      required: false
-  - type: input
-    id: python_version
-    attributes:
-      label: Python version
-      description: Use `python3 --version`
-    validations:
-      required: false
-  - type: input
-    id: database_version
-    attributes:
-      label: Database version (if applicable)
-      description: Use `psql --version` / `mysql --version` / `mongodump --version` / `sqlite3 --version`
-    validations:
-      required: false
-  - type: input
-    id: operating_system_version
-    attributes:
-      label: Operating system and version
-      description: On Linux, use `cat /etc/os-release`
-    validations:
-      required: false

.gitea/issue_template/config.yaml

@@ -1 +0,0 @@
-blank_issues_enabled: true

.gitea/issue_template/feature_template.yaml

@@ -1,15 +0,0 @@
-name: "Feature"
-about: "For filing a feature request or idea"
-body:
-  - type: textarea
-    id: request
-    attributes:
-      label: What I'd like to do and why
-    validations:
-      required: true
-  - type: textarea
-    id: notes
-    attributes:
-      label: Other notes / implementation ideas
-    validations:
-      required: false

.gitea/workflows/build.yaml

@@ -1,30 +0,0 @@
-name: build
-run-name: ${{ gitea.actor }} is building
-on:
-  push:
-    branches: [main]
-
-jobs:
-  test:
-    runs-on: host
-
-    steps:
-      - uses: actions/checkout@v4
-      - run: scripts/run-end-to-end-tests
-
-  docs:
-    needs: [test]
-    runs-on: host
-    env:
-      IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
-
-    steps:
-      - uses: actions/checkout@v4
-      - run: podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org
-        env:
-          USERNAME: "${{ secrets.REGISTRY_USERNAME }}"
-          PASSWORD: "${{ secrets.REGISTRY_PASSWORD }}"
-      - run: podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
-      - run: podman push "$IMAGE_NAME"
-      - run: scripts/export-docs-from-image
-      - run: curl --user "${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }}" --upload-file borgmatic-docs.tar.gz https://projects.torsion.org/api/packages/borgmatic-collective/generic/borgmatic-docs/$(head --lines=1 NEWS)/borgmatic-docs.tar.gz

NEWS

@@ -1,739 +1,3 @@
-2.0.0.dev0
- * #262: Add a "default_actions" option that supports disabling default actions when borgmatic is
-   run without any command-line arguments.
- * #345: Add a "key import" action to import a repository key from backup.
- * #422: Add home directory expansion to file-based and KeePassXC credential hooks.
- * #610: Add a "recreate" action for recreating archives, for instance for retroactively excluding
-   particular files from existing archives.
- * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more
-   flexible "commands:". See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
- * #790: BREAKING: For both new and deprecated command hooks, run a configured "after" hook even if
-   an error occurs first. This allows you to perform cleanup steps that correspond to "before"
-   preparation commands—even when something goes wrong.
- * #790: BREAKING: Run all command hooks (both new and deprecated) respecting the
-   "working_directory" option if configured, meaning that hook commands are run in that directory.
- * #836: Add a custom command option for the SQLite hook.
- * #837: Add custom command options for the MongoDB hook.
- * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune".
- * #1020: Document a database use case involving a temporary database client container:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
- * #1037: Fix an error with the "extract" action when both a remote repository and a
-   "working_directory" are used.
- * #1044: Fix an error in the systemd credential hook when the credential name contains a "."
-   character.
-
-1.9.14
- * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the
-   incident UI. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook
- * #936: Clarify Zabbix monitoring hook documentation about creating items:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook
- * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly.
- * #1021: Fix a regression in which the "exclude_patterns" option didn't expand "~" (the user's
-   home directory). This fix means that all "patterns" and "patterns_from" also now expand "~".
- * #1023: Fix an error in the Btrfs hook when attempting to snapshot a read-only subvolume. Now,
-   read-only subvolumes are ignored since Btrfs can't actually snapshot them.
-
-1.9.13
- * #975: Add a "compression" option to the PostgreSQL database hook.
- * #1001: Fix a ZFS error during snapshot cleanup.
- * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes.
- * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than
-   using an environment variable.
- * #1013: Send database passwords to MongoDB via anonymous pipe, which is more secure than using
-   "--password" on the command-line!
- * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit.
- * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification.
- * Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption
-   between client and server.
-
-1.9.12
- * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will
-   work with Python 3.9, 3.10, and 3.11 again.
-
-1.9.11
- * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
- * #996: Fix the "create" action to omit the repository label prefix from Borg's output when
-   databases are enabled.
- * #998: Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure
-   than using an environment variable.
- * #999: Fix a runtime directory error from a conflict between "extra_borg_options" and special file
-   detection.
- * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from
-   a borgmatic configuration option (e.g. "source_directories")—not from other hooks within
-   borgmatic.
- * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical
-   volumes.
- * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property.
- * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
-   source directories.
-
-1.9.10
- * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic
-   configuration files. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
- * #987: Fix a "list" action error when the "encryption_passcommand" option is set.
- * #987: When both "encryption_passcommand" and "encryption_passphrase" are configured, prefer
-   "encryption_passphrase" even if it's an empty value.
- * #988: With the "max_duration" option or the "--max-duration" flag, run the archives and
-   repository checks separately so they don't interfere with one another. Previously, borgmatic
-   refused to run checks in this situation.
- * #989: Fix the log message code to avoid using Python 3.10+ logging features. Now borgmatic will
-   work with Python 3.9 again.
- * Capture and delay any log records produced before logging is fully configured, so early log
-   records don't get lost.
- * Add support for Python 3.13.
-
-1.9.9
- * #635: Log the repository path or label on every relevant log message, not just some logs.
- * #961: When the "encryption_passcommand" option is set, call the command once from borgmatic to
-   collect the encryption passphrase and then pass it to Borg multiple times. See the documentation
-   for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
- * #981: Fix a "spot" check file count delta error.
- * #982: Fix for borgmatic "exclude_patterns" and "exclude_from" recursing into excluded
-   subdirectories.
- * #983: Fix the Btrfs hook to support subvolumes with names like "@home" different from their
-   mount points.
- * #985: Change the default value for the "--original-hostname" flag from "localhost" to no host
-   specified. This way, the "restore" action works without a hostname if there's a single matching
-   database dump.
-
-1.9.8
- * #979: Fix root patterns so they don't have an invalid "sh:" prefix before getting passed to Borg.
- * Expand the recent contributors documentation section to include ticket submitters—not just code
-   contributors—because there are multiple ways to contribute to the project! See:
-   https://torsion.org/borgmatic/#recent-contributors
-
-1.9.7
- * #855: Add a Sentry monitoring hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook
- * #968: Fix for a "spot" check error when a filename in the most recent archive contains a newline.
- * #970: Fix for an error when there's a blank line in the configured patterns or excludes.
- * #971: Fix for "exclude_from" files being completely ignored.
- * #977: Fix for "exclude_patterns" and "exclude_from" not supporting explicit pattern styles (e.g.,
-   "sh:" or "re:").
-
-1.9.6
- * #959: Fix an error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
-   source directories.
- * #960: Fix for archives storing relative source directory paths such that they contain the working
-   directory.
- * #960: Fix the "spot" check to support relative source directory paths.
- * #962: For the ZFS, Btrfs, and LVM hooks, perform path rewriting for excludes and patterns in
-   addition to the existing source directories rewriting.
- * #962: Under the hood, merge all configured source directories, excludes, and patterns into a
-   unified temporary patterns file for passing to Borg. The borgmatic configuration options remain
-   unchanged.
- * #962: For the LVM hook, add support for nested logical volumes.
- * #965: Fix a borgmatic runtime directory error when running the "spot" check with a database hook
-   enabled.
- * #969: Fix the "restore" action to work on database dumps without a port when a default port is
-   present in configuration.
- * Fix the "spot" check to no longer consider pipe files within an archive for file comparisons.
- * Fix the "spot" check to have a nicer error when there are no source paths to compare.
- * Fix auto-excluding of special files (when databases are configured) to support relative source
-   directory paths.
- * Drop support for Python 3.8, which has been end-of-lifed.
-
-1.9.5
- * #418: Backup and restore databases that have the same name but with different ports, hostnames,
-   or hooks.
- * #947: To avoid a hang in the database hooks, error and exit when the borgmatic runtime
-   directory overlaps with the configured excludes.
- * #954: Fix a findmnt command error in the Btrfs hook by switching to parsing JSON output.
- * #956: Fix the printing of a color reset code even when color is disabled.
- * #958: Drop colorama as a library dependency.
- * When the ZFS, Btrfs, or LVM hooks aren't configured, don't try to cleanup snapshots for them.
-
-1.9.4
- * #80 (beta): Add an LVM hook for snapshotting and backing up LVM logical volumes. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
- * #251 (beta): Add a Btrfs hook for snapshotting and backing up Btrfs subvolumes. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
- * #926: Fix a library error when running within a PyInstaller bundle.
- * #950: Fix a snapshot unmount error in the ZFS hook when using nested datasets.
- * Update the ZFS hook to discover and snapshot ZFS datasets even if they are parent/grandparent
-   directories of your source directories.
- * Reorganize data source and monitoring hooks to make developing new hooks easier.
-
-1.9.3
- * #261 (beta): Add a ZFS hook for snapshotting and backing up ZFS datasets. See the documentation
-   for more information: https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
- * Remove any temporary copies of the manifest file created in support of the "bootstrap" action.
- * Deprecate the "store_config_files" option at the global scope and move it under the "bootstrap"
-   hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive
- * Require the runtime directory to be an absolute path.
- * Add a "--deleted" flag to the "repo-list" action for listing deleted archives that haven't
-   yet been compacted (Borg 2 only).
- * Promote the "spot" check from a beta feature to stable.
-
-1.9.2
- * #441: Apply the "umask" option to all relevant actions, not just some of them.
- * #722: Remove the restriction that the "extract" and "mount" actions must match a single
-   repository. Now they work more like other actions, where each repository is applied in turn.
- * #932: Fix the missing build backend setting in pyproject.toml to allow Fedora builds.
- * #934: Update the logic that probes for the borgmatic streaming database dump, bootstrap
-   metadata, and check state directories to support more platforms and use cases. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory
- * #934: Add the "RuntimeDirectory" and "StateDirectory" options to the sample systemd service
-   file to support the new runtime and state directory logic.
- * #939: Fix borgmatic ignoring the "BORG_RELOCATED_REPO_ACCESS_IS_OK" and
-   "BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK" environment variables.
- * Add a Pushover monitoring hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook
-
-1.9.1
- * #928: Fix the user runtime directory location on macOS (and possibly Cygwin).
- * #930: Fix an error with the sample systemd service when no credentials are configured.
- * #931: Fix an error when implicitly upgrading the check state directory from ~/.borgmatic to
-   ~/.local/state/borgmatic across filesystems.
-
-1.9.0
- * #609: Fix the glob expansion of "source_directories" values to respect the "working_directory"
-   option.
- * #609: BREAKING: Apply the "working_directory" option to all actions, not just "create". This
-   includes repository paths, destination paths, mount points, etc.
- * #562: Deprecate the "borgmatic_source_directory" option in favor of "user_runtime_directory"
-   and "user_state_directory".
- * #562: BREAKING: Move the default borgmatic streaming database dump and bootstrap metadata
-   directory from ~/.borgmatic to /run/user/$UID/borgmatic, which is more XDG-compliant. You can
-   override this location with the new "user_runtime_directory" option. Existing archives with
-   database dumps at the old location are still restorable. 
- * #562, #638: Move the default check state directory from ~/.borgmatic to 
-   ~/.local/state/borgmatic. This is more XDG-compliant and also prevents these state files from
-   getting backed up (unless you explicitly include them). You can override this location with the
-   new "user_state_directory" option. After the first time you run the "check" action with borgmatic
-   1.9.0, you can safely delete the ~/.borgmatic directory.
- * #838: BREAKING: With Borg 1.4+, store database dumps and bootstrap metadata in a "/borgmatic"
-   directory within a backup archive, so the path doesn't depend on the current user. This means
-   that you can now backup as one user and restore or bootstrap as another user, among other use
-   cases.
- * #902: Add loading of encrypted systemd credentials. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/#using-systemd-service-credentials
- * #911: Add a "key change-passphrase" action to change the passphrase protecting a repository key.
- * #914: Fix a confusing apparent hang when when the repository location changes, and instead
-   show a helpful error message.
- * #915: BREAKING: Rename repository actions like "rcreate" to more explicit names like
-   "repo-create" for compatibility with recent changes in Borg 2.0.0b10.
- * #918: BREAKING: When databases are configured, don't auto-enable the "one_file_system" option,
-   as existing auto-excludes of special files should be sufficient to prevent Borg from hanging on
-   them. But if this change causes problems for you, you can always enable "one_file_system"
-   explicitly.
- * #919: Clarify the command-line help for the "--config" flag.
- * #919: Document a policy for versioning and breaking changes:
-   https://torsion.org/borgmatic/docs/how-to/upgrade/#versioning-and-breaking-changes
- * #921: BREAKING: Change soft failure command hooks to skip only the current repository rather than
-   all repositories in the configuration file.
- * #922: Replace setup.py (Python packaging metadata) with the more modern pyproject.toml.
- * When using Borg 2, default the "archive_name_format" option to just "{hostname}", as Borg 2 does
-   not require unique archive names; identical archive names form a common "series" that can be
-   targeted together. See the Borg 2 documentation for more information:
-   https://borgbackup.readthedocs.io/en/2.0.0b13/changes.html#borg-1-2-x-1-4-x-to-borg-2-0
- * Add support for Borg 2's "rclone:" repository URLs, so you can backup to 70+ cloud storage
-   services whether or not they support Borg explicitly.
- * Add support for Borg 2's "sftp://" repository URLs.
- * Update the "--match-archives" and "--archive" flags to support Borg 2 series names or archive
-   hashes.
- * Add a "--match-archives" flag to the "prune" action.
- * Add "--local-path" and "--remote-path" flags to the "config bootstrap" action for setting the
-   Borg executable paths used for bootstrapping.
- * Add a "--user-runtime-directory" flag to the "config bootstrap" action for helping borgmatic
-   locate the bootstrap metadata stored in an archive.
- * Add a Zabbix monitoring hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook
- * Add a tarball of borgmatic's HTML documentation to the packages on the project page.
-
-1.8.14
- * #896: Fix an error in borgmatic rcreate/init on an empty repository directory with Borg 1.4.
- * #898: Add glob ("*") support to the "--repository" flag. Just quote any values containing
-   globs so your shell doesn't interpret them.
- * #899: Fix for a "bad character" Borg error in which the "spot" check fed Borg an invalid pattern.
- * #900: Fix for a potential traceback (TypeError) during the handling of another error.
- * #904: Clarify the configuration reference about the "spot" check options:
-   https://torsion.org/borgmatic/docs/reference/configuration/
- * #905: Fix the "source_directories_must_exist" option to work with relative "source_directories"
-   paths when a "working_directory" is set.
- * #906: Add documentation details for how to run custom database dump commands using binaries from
-   running containers:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
- * Fix a regression in which the "color" option had no effect.
- * Add a recent contributors section to the documentation, because credit where credit's due! See:
-   https://torsion.org/borgmatic/#recent-contributors
-
-1.8.13
- * #298: Add "delete" and "rdelete" actions to delete archives or entire repositories.
- * #785: Add an "only_run_on" option to consistency checks so you can limit a check to running on
-   particular days of the week. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#check-days
- * #885: Add an Uptime Kuma monitoring hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook
- * #886: Fix a PagerDuty hook traceback with Python < 3.10.
- * #889: Fix the Healthchecks ping body size limit, restoring it to the documented 100,000 bytes.
-
-1.8.12
- * #817: Add a "--max-duration" flag to the "check" action and a "max_duration" option to the
-   repository check configuration. This tells Borg to interrupt a repository check after a certain
-   duration.
- * #860: Fix interaction between environment variable interpolation in constants and shell escaping.
- * #863: When color output is disabled (explicitly or implicitly), don't prefix each log line with
-   the log level.
- * #865: Add an "upload_buffer_size" option to set the size of the upload buffer used in "create"
-   action.
- * #866: Fix "Argument list too long" error in the "spot" check when checking hundreds of thousands
-   of files at once.
- * #874: Add the configured repository label as "repository_label" to the interpolated variables
-   passed to before/after command hooks.
- * #881: Fix "Unrecognized argument" error when the same value is used with different command-line
-   flags.
- * In the "spot" check, don't try to hash symlinked directories.
-
-1.8.11
- * #815: Add optional Healthchecks auto-provisioning via "create_slug" option.
- * #851: Fix lack of file extraction when using "extract --strip-components all" on a path with a
-   leading slash.
- * #854: Fix a traceback when the "data" consistency check is used.
- * #857: Fix a traceback with "check --only spot" when the "spot" check is unconfigured.
-
-1.8.10
- * #656 (beta): Add a "spot" consistency check that compares file counts and contents between your
-   source files and the latest archive, ensuring they fall within configured tolerances. This can
-   catch problems like incorrect excludes, inadvertent deletes, files changed by malware, etc. See
-   the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#spot-check
- * #779: When "--match-archives *" is used with "check" action, don't skip Borg's orphaned objects
-   check.
- * #842: When a command hook exits with a soft failure, ping the log and finish states for any
-   configured monitoring hooks.
- * #843: Add documentation link to Loki dashboard for borgmatic:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook
- * #847: Fix "--json" error when Borg includes non-JSON warnings in JSON output.
- * #848: SECURITY: Mask the password when logging a MongoDB dump or restore command.
- * Fix handling of the NO_COLOR environment variable to ignore an empty value.
- * Add documentation about backing up containerized databases by configuring borgmatic to exec into
-   a container to run a dump command:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
-
-1.8.9
- * #311: Add custom dump/restore command options for MySQL and MariaDB.
- * #811: Add an "access_token" option to the ntfy monitoring hook for authenticating
-   without username/password.
- * #827: When the "--json" flag is given, suppress console escape codes so as not to
-   interfere with JSON output.
- * #829: Fix "--override" values containing deprecated section headers not actually overriding
-   configuration options under deprecated section headers.
- * #835: Add support for the NO_COLOR environment variable. See the documentation for more
-   information:
-   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#colored-output
- * #839: Add log sending for the Apprise logging hook, enabled by default. See the documentation for
-   more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook
- * #839: Document a potentially breaking shell quoting edge case within error hooks:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks
- * #840: When running the "rcreate" action and the repository already exists but with a different
-   encryption mode than requested, error.
- * Switch from Drone to Gitea Actions for continuous integration.
- * Rename scripts/run-end-to-end-dev-tests to scripts/run-end-to-end-tests and use it in both dev
-   and CI for better dev-CI parity.
- * Clarify documentation about restoring a database: borgmatic does not create the database upon
-   restore.
-
-1.8.8
- * #370: For the PostgreSQL hook, pass the "PGSSLMODE" environment variable through to Borg when the
-   database's configuration omits the "ssl_mode" option.
- * #818: Allow the "--repository" flag to match across multiple configuration files.
- * #820: Fix broken repository detection in the "rcreate" action with Borg 1.4. The issue did not
-   occur with other versions of Borg.
- * #822: Fix broken escaping logic in the PostgreSQL hook's "pg_dump_command" option.
- * SECURITY: Prevent additional shell injection attacks within the PostgreSQL hook.
-
-1.8.7
- * #736: Store included configuration files within each backup archive in support of the "config
-   bootstrap" action. Previously, only top-level configuration files were stored.
- * #798: Elevate specific Borg warnings to errors or squash errors to
- * warnings. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/customize-warnings-and-errors/
- * #810: SECURITY: Prevent shell injection attacks within the PostgreSQL hook, the MongoDB hook, the
-   SQLite hook, the "borgmatic borg" action, and command hook variable/constant interpolation.
- * #814: Fix a traceback when providing an invalid "--override" value for a list option.
-
-1.8.6
- * #767: Add an "--ssh-command" flag to the "config bootstrap" action for setting a custom SSH
-   command, as no configuration is available (including the "ssh_command" option) until
-   bootstrapping completes.
- * #794: Fix a traceback when the "repositories" option contains both strings and key/value pairs.
- * #800: Add configured repository labels to the JSON output for all actions.
- * #802: The "check --force" flag now runs checks even if "check" is in "skip_actions".
- * #804: Validate the configured action names in the "skip_actions" option.
- * #807: Stream SQLite databases directly to Borg instead of dumping to an intermediate file.
- * When logging commands that borgmatic executes, log the environment variables that
-   borgmatic sets for those commands. (But don't log their values, since they often contain
-   passwords.)
-
-1.8.5
- * #701: Add a "skip_actions" option to skip running particular actions, handy for append-only or
-   checkless configurations. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#skipping-actions
- * #701: Deprecate the "disabled" value for the "checks" option in favor of the new "skip_actions"
-   option.
- * #745: Constants now apply to included configuration, not just the file doing the includes. As a
-   side effect of this change, constants no longer apply to option names and only substitute into
-   configuration values.
- * #779: Add a "--match-archives" flag to the "check" action for selecting the archives to check,
-   overriding the existing "archive_name_format" and "match_archives" options in configuration.
- * #779: Only parse "--override" values as complex data types when they're for options of those
-   types.
- * #782: Fix environment variable interpolation within configured repository paths.
- * #782: Add configuration constant overriding via the existing "--override" flag.
- * #783: Upgrade ruamel.yaml dependency to support version 0.18.x.
- * #784: Drop support for Python 3.7, which has been end-of-lifed.
-
-1.8.4
- * #715: Add a monitoring hook for sending backup status to a variety of monitoring services via the
-   Apprise library. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook
- * #748: When an archive filter causes no matching archives for the "rlist" or "info"
-   actions, warn the user and suggest how to remove the filter.
- * #768: Fix a traceback when an invalid command-line flag or action is used.
- * #771: Fix normalization of deprecated sections ("location:", "storage:", "hooks:", etc.) to
-   support empty sections without erroring.
- * #774: Disallow the "--dry-run" flag with the "borg" action, as borgmatic can't guarantee the Borg
-   command won't have side effects.
-
-1.8.3
- * #665: BREAKING: Simplify logging logic as follows: Syslog verbosity is now disabled by
-   default, but setting the "--syslog-verbosity" flag enables it regardless of whether you're at an
-   interactive console. Additionally, "--log-file-verbosity" and "--monitoring-verbosity" now
-   default to 1 (info about steps borgmatic is taking) instead of 0. And both syslog logging and
-   file logging can be enabled simultaneously.
- * #743: Add a monitoring hook for sending backup status and logs to Grafana Loki. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook
- * #753: When "archive_name_format" is not set, filter archives using the default archive name
-   format.
- * #754: Fix error handling to log command output as one record per line instead of truncating
-   too-long output and swallowing the end of some Borg error messages.
- * #757: Update documentation so "sudo borgmatic" works for pipx borgmatic installations.
- * #761: Fix for borgmatic not stopping Borg immediately when the user presses ctrl-C.
- * Update documentation to recommend installing/upgrading borgmatic with pipx instead of pip. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#installation
-   https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-borgmatic
-
-1.8.2
- * #345: Add "key export" action to export a copy of the repository key for safekeeping in case
-   the original goes missing or gets damaged.
- * #727: Add a MariaDB database hook that uses native MariaDB commands instead of the deprecated
-   MySQL ones. Be aware though that any existing backups made with the "mysql_databases:" hook are
-   only restorable with a "mysql_databases:" configuration.
- * #738: Fix for potential data loss (data not getting restored) in which the database "restore"
-   action didn't actually restore anything and indicated success anyway.
- * Remove the deprecated use of the MongoDB hook's "--db" flag for database restoration.
- * Add source code reference documentation for getting oriented with the borgmatic code as a
-   developer: https://torsion.org/borgmatic/docs/reference/source-code/
-
-1.8.1
- * #326: Add documentation for restoring a database to an alternate host:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-to-an-alternate-host
- * #697: Add documentation for "bootstrap" action:
-   https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive
- * #725: Add "store_config_files" option for disabling the automatic backup of configuration files
-   used by the "config bootstrap" action.
- * #728: Fix for "prune" action error when using the "keep_exclude_tags" option.
- * #730: Fix for Borg's interactive prompt on the "check --repair" action automatically getting
-   answered "NO" even when the "check_i_know_what_i_am_doing" option isn't set.
- * #732: Include multiple configuration files with a single "!include". See the documentation for
-   more information:
-   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#multiple-merge-includes
- * #734: Omit "--glob-archives" or "--match-archives" Borg flag when its value would be "*" (meaning
-   all archives).
-
-1.8.0
- * #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting
-   repository/archive into the resulting Borg command-line, pass repository to Borg via an
-   environment variable and make archive available for explicit use in your commands. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/
- * #719: Fix an error when running "borg key export" through borgmatic.
- * #720: Fix an error when dumping a database and the "exclude_nodump" option is set.
- * #724: Add "check_i_know_what_i_am_doing" option to bypass Borg confirmation prompt when running
-   "check --repair".
- * When merging two configuration files, error gracefully if the two files do not adhere to the same
-   format.
- * #721: Remove configuration sections ("location:", "storage:", "hooks:", etc.), while still
-   keeping deprecated support for them. Now, all options are at the same level, and you don't need
-   to worry about commenting/uncommenting section headers when you change an option (if you remove
-   your sections first).
- * #721: BREAKING: The retention prefix and the consistency prefix can no longer have different
-   values (unless one is not set).
- * #721: BREAKING: The storage umask and the hooks umask can no longer have different values (unless
-   one is not set).
- * BREAKING: Flags like "--config" that previously took multiple values now need to be given once
-   per value, e.g. "--config first.yaml --config second.yaml" instead of "--config first.yaml
-   second.yaml". This prevents argument parsing errors on ambiguous commands.
- * BREAKING: Remove the deprecated (and silently ignored) "--successful" flag on the "list" action,
-   as newer versions of Borg list successful (non-checkpoint) archives by default.
- * All deprecated configuration option values now generate warning logs.
- * Remove the deprecated (and non-functional) "--excludes" flag in favor of excludes within
-   configuration.
- * Fix an error when logging too-long command output during error handling. Now, long command output
-   is truncated before logging.
-
-1.7.15
- * #326: Add configuration options and command-line flags for backing up a database from one
-   location while restoring it somewhere else.
- * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors.
- * #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style
-   configuration.
- * #529: Deprecate generate-borgmatic-config in favor of new "config generate" action.
- * #529: Deprecate validate-borgmatic-config in favor of new "config validate" action.
- * #697, #712, #716: Extract borgmatic configuration from backup via new "config bootstrap"
-   action—even when borgmatic has no configuration yet!
- * #669: Add sample systemd user service for running borgmatic as a non-root user.
- * #711, #713: Fix an error when "data" check time files are accessed without getting upgraded
-   first.
-
-1.7.14
- * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file,
-   or monitoring), so not even errors are shown.
- * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist.
- * #659: Add Borg 2 date-based matching flags to various actions for archive selection.
- * #703: Fix an error when loading the configuration schema on Fedora Linux.
- * #704: Fix "check" action error when repository and archive checks are configured but the archive
-   check gets skipped due to the configured frequency.
- * #706: Fix "--archive latest" on "list" and "info" actions that only worked on the first of
-   multiple configured repositories.
-
-1.7.13
- * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema"
-   flag. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas
- * #678: Fix error from PostgreSQL when dumping a database with a "format" of "plain".
- * #678: Fix PostgreSQL hook to support "psql_command" and "pg_restore_command" options containing
-   commands with arguments.
- * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break
-   database dumping.
- * #680: Add support for logging each log line as a JSON object via global "--log-json" flag.
- * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories.
- * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to
-   update your development checkouts accordingly.
- * #686: Add fish shell completion script so you can tab-complete on the borgmatic command-line. See
-   the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion
- * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install
-   --editable" development installs.
- * #688: Fix archive checks being skipped even when particular archives haven't been checked
-   recently. This occurred when using multiple borgmatic configuration files with different
-   "archive_name_format"s, for instance.
- * #691: Fix error in "borgmatic restore" action when the configured repository path is relative
-   instead of absolute.
- * #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like
-   "--progress" still work.
-
-1.7.12
- * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file.
-   See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
- * #666, #670: Fix error when running the "info" action with the "--match-archives" or "--archive"
-   flags. Also fix the "--match-archives"/"--archive" flags to correctly override the
-   "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions.
- * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix"
-   options set.
- * #672: Selectively shallow merge certain mappings or sequences when including configuration files.
-   See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge
- * #672: Selectively omit list values when including configuration files. See the documentation for
-   more information:
-   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#list-merge
- * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag.
-   See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes
- * Add optional support for running end-to-end tests and building documentation with rootless Podman
-   instead of Docker.
-
-1.7.11
- * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives
-   get used for borgmatic actions that operate on multiple archives. Override this behavior with the
-   new "match_archives" option in the storage section. This change is "breaking" in that it silently
-   changes which archives get considered for "rlist", "prune", "check", etc. See the documentation
-   for more information:
-   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming
- * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format"
-   auto-matching behavior and the "match_archives" option.
- * #658: Add "--log-file-format" flag for customizing the log message format. See the documentation
-   for more information:
-   https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#logging-to-file
- * #662: Fix regression in which the "check_repositories" option failed to match repositories.
- * #663: Fix regression in which the "transfer" action produced a traceback.
- * Add spellchecking of source code during test runs.
-
-1.7.10
- * #396: When a database command errors, display and log the error message instead of swallowing it.
- * #501: Optionally error if a source directory does not exist via "source_directories_must_exist"
-   option in borgmatic's location configuration.
- * #576: Add support for "file://" paths within "repositories" option.
- * #612: Define and use custom constants in borgmatic configuration files. See the documentation for
-   more information:
-   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#constant-interpolation
- * #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option
-   in borgmatic's storage configuration.
- * #623: Fix confusing message when an error occurs running actions for a configuration file.
- * #635: Add optional repository labels so you can select a repository via "--repository yourlabel"
-   at the command-line. See the configuration reference for more information:
-   https://torsion.org/borgmatic/docs/reference/configuration/
- * #649: Add documentation on backing up a database running in a container:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
- * #655: Fix error when databases are configured and a source directory doesn't exist.
- * Add code style plugins to enforce use of Python f-strings and prevent single-letter variables.
-   To join in the pedantry, refresh your test environment with "tox --recreate".
- * Rename scripts/run-full-dev-tests to scripts/run-end-to-end-dev-tests and make it run end-to-end
-   tests only. Continue using tox to run unit and integration tests.
-
-1.7.9
- * #295: Add a SQLite database dump/restore hook.
- * #304: Change the default action order when no actions are specified on the command-line to:
-   "create", "prune", "compact", "check". If you'd like to retain the old ordering ("prune" and
-   "compact" first), then specify actions explicitly on the command-line.
- * #304: Run any command-line actions in the order specified instead of using a fixed ordering.
- * #564: Add "--repository" flag to all actions where it makes sense, so you can run borgmatic on
-   a single configured repository instead of all of them.
- * #628: Add a Healthchecks "log" state to send borgmatic logs to Healthchecks without signalling
-   success or failure.
- * #647: Add "--strip-components all" feature on the "extract" action to remove leading path
-   components of files you extract. Must be used with the "--path" flag.
- * Add support for Python 3.11.
-
-1.7.8
- * #620: With the "create" action and the "--list" ("--files") flag, only show excluded files at
-   verbosity 2.
- * #621: Add optional authentication to the ntfy monitoring hook.
- * With the "create" action, only one of "--list" ("--files") and "--progress" flags can be used.
-   This lines up with the new behavior in Borg 2.0.0b5.
- * Internally support new Borg 2.0.0b5 "--filter" status characters / item flags for the "create"
-   action.
- * Fix the "create" action with the "--dry-run" flag querying for databases when a PostgreSQL/MySQL
-   "all" database is configured. Now, these queries are skipped due to the dry run.
- * Add "--repository" flag to the "rcreate" action to optionally select one configured repository to
-   create.
- * Add "--progress" flag to the "transfer" action, new in Borg 2.0.0b5.
- * Add "checkpoint_volume" configuration option to creates checkpoints every specified number of
-   bytes during a long-running backup, new in Borg 2.0.0b5.
-
-1.7.7
- * #642: Add MySQL database hook "add_drop_database" configuration option to control whether dumped
-   MySQL databases get dropped right before restore.
- * #643: Fix for potential data loss (data not getting backed up) when dumping large "directory"
-   format PostgreSQL/MongoDB databases. Prior to the fix, these dumps would not finish writing to
-   disk before Borg consumed them. Now, the dumping process completes before Borg starts. This only
-   applies to "directory" format databases; other formats still stream to Borg without using
-   temporary disk space.
- * Fix MongoDB "directory" format to work with mongodump/mongorestore without error. Prior to this
-   fix, only the "archive" format worked.
-
-1.7.6
- * #393, #438, #560: Optionally dump "all" PostgreSQL/MySQL databases to separate files instead of
-   one combined dump file, allowing more convenient restores of individual databases. You can enable
-   this by specifying the database dump "format" option when the database is named "all".
- * #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout.
- * #622: Fix traceback when include merging configuration files on ARM64.
- * #629: Skip warning about excluded special files when no special files have been excluded.
- * #630: Add configuration options for database command customization: "list_options",
-   "restore_options", and "analyze_options" for PostgreSQL, "restore_options" for MySQL, and
-   "restore_options" for MongoDB.
-
-1.7.5
- * #311: Override PostgreSQL dump/restore commands via configuration options.
- * #604: Fix traceback when a configuration section is present but lacking any options.
- * #607: Clarify documentation examples for include merging and deep merging.
- * #611: Fix "data" consistency check to support "check_last" and consistency "prefix" options.
- * #613: Clarify documentation about multiple repositories and separate configuration files.
-
-1.7.4
- * #596: Fix special file detection erroring when broken symlinks are encountered.
- * #597, #598: Fix regression in which "check" action errored on certain systems ("Cannot determine
-   Borg repository ID").
-
-1.7.3
- * #357: Add "break-lock" action for removing any repository and cache locks leftover from Borg
-   aborting.
- * #360: To prevent Borg hangs, unconditionally delete stale named pipes before dumping databases.
- * #587: When database hooks are enabled, auto-exclude special files from a "create" action to
-   prevent Borg from hanging. You can override/prevent this behavior by explicitly setting the
-   "read_special" option to true.
- * #587: Warn when ignoring a configured "read_special" value of false, as true is needed when
-   database hooks are enabled.
- * #589: Update sample systemd service file to allow system "idle" (e.g. a video monitor turning
-   off) while borgmatic is running.
- * #590: Fix for potential data loss (data not getting backed up) when the "patterns_from" option
-   was used with "source_directories" (or the "~/.borgmatic" path existed, which got injected into
-   "source_directories" implicitly). The fix is for borgmatic to convert "source_directories" into
-   patterns whenever "patterns_from" is used, working around a Borg bug:
-   https://github.com/borgbackup/borg/issues/6994
- * #590: In "borgmatic create --list" output, display which files get excluded from the backup due
-   to patterns or excludes.
- * #591: Add support for Borg 2's "--match-archives" flag. This replaces "--glob-archives", which
-   borgmatic now treats as an alias for "--match-archives". But note that the two flags have
-   slightly different syntax. See the Borg 2 changelog for more information:
-   https://borgbackup.readthedocs.io/en/2.0.0b3/changes.html#version-2-0-0b3-2022-10-02
- * Fix for "borgmatic --archive latest" not finding the latest archive when a verbosity is set.
-
-1.7.2
- * #577: Fix regression in which "borgmatic info --archive ..." showed repository info instead of
-   archive info with Borg 1.
- * #582: Fix hang when database hooks are enabled and "patterns" contains a parent directory of
-   "~/.borgmatic".
-
-1.7.1
- * #542: Make the "source_directories" option optional. This is useful for "check"-only setups or
-   using "patterns" exclusively.
- * #574: Fix for potential data loss (data not getting backed up) when the "patterns" option was
-   used with "source_directories" (or the "~/.borgmatic" path existed, which got injected into
-   "source_directories" implicitly). The fix is for borgmatic to convert "source_directories" into
-   patterns whenever "patterns" is used, working around a Borg bug:
-   https://github.com/borgbackup/borg/issues/6994
-
-1.7.0
- * #463: Add "before_actions" and "after_actions" command hooks that run before/after all the
-   actions for each repository. These new hooks are a good place to run per-repository steps like
-   mounting/unmounting a remote filesystem.
- * #463: Update documentation to cover per-repository configurations:
-   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/
- * #557: Support for Borg 2 while still working with Borg 1. This includes new borgmatic actions
-   like "rcreate" (replaces "init"), "rlist" (list archives in repository), "rinfo" (show repository
-   info), and "transfer" (for upgrading Borg repositories). For the most part, borgmatic tries to
-   smooth over differences between Borg 1 and 2 to make your upgrade process easier. However, there
-   are still a few cases where Borg made breaking changes. See the Borg 2.0 changelog for more
-   information: https://www.borgbackup.org/releases/borg-2.0.html
- * #557: If you install Borg 2, you'll need to manually upgrade your existing Borg 1 repositories
-   before use. Note that Borg 2 stable is not yet released as of this borgmatic release, so don't
-   use Borg 2 for production until it is! See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-borg
- * #557: Rename several configuration options to match Borg 2: "remote_rate_limit" is now
-   "upload_rate_limit", "numeric_owner" is "numeric_ids", and "bsd_flags" is "flags". borgmatic
-   still works with the old options.
- * #557: Remote repository paths without the "ssh://" syntax are deprecated but still supported for
-   now. Remote repository paths containing "~" are deprecated in borgmatic and no longer work in
-   Borg 2.
- * #557: Omitting the "--archive" flag on the "list" action is deprecated when using Borg 2. Use
-   the new "rlist" action instead.
- * #557: The "--dry-run" flag can now be used with the "rcreate"/"init" action.
- * #565: Fix handling of "repository" and "data" consistency checks to prevent invalid Borg flags.
- * #566: Modify "mount" and "extract" actions to require the "--repository" flag when multiple
-   repositories are configured.
- * #571: BREAKING: Remove old-style command-line action flags like "--create, "--list", etc. If
-   you're already using actions like "create" and "list" instead, this change should not affect you.
- * #571: BREAKING: Rename "--files" flag on "prune" action to "--list", as it lists archives, not
-   files.
- * #571: Add "--list" as alias for "--files" flag on "create" and "export-tar" actions.
- * Add support for disabling TLS verification in Healthchecks monitoring hook with "verify_tls"
-   option.
-
 1.6.6
  * #559: Update documentation about configuring multiple consistency checks or multiple databases.
  * #560: Fix all database hooks to error when the requested database to restore isn't present in the
@@ -900,7 +164,7 @@
  * #398: Clarify canonical home of borgmatic in documentation.
  * #406: Clarify that spaces in path names should not be backslashed in path names.
  * #423: Fix error handling to error loudly when Borg gets killed due to running out of memory!
- * Fix build so as not to attempt to build and push documentation for a non-main branch.
+ * Fix build so as not to attempt to build and push documentation for a non-master branch.
  * "Fix" build failure with Alpine Edge by switching from Edge to Alpine 3.13.
  * Move #borgmatic IRC channel from Freenode to Libera Chat due to Freenode takeover drama.
    IRC connection info: https://torsion.org/borgmatic/#issues
@@ -963,7 +227,7 @@
    configuration schema descriptions.
 
 1.5.6
- * #292: Allow before_backup and similar hooks to exit with a soft failure without altering the
+ * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the
    monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring
    services with a "start" status until after before_* hooks finish. Failures in before_* hooks
    still trigger a monitoring "fail" status.
@@ -1032,7 +296,7 @@
  * For "list" and "info" actions, show repository names even at verbosity 0.
 
 1.4.22
- * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON output.
+ * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON ouput.
  * After a backup of a database dump in directory format, properly remove the dump directory.
  * In "borgmatic --help", don't expand $HOME in listing of default "--config" paths.
 
@@ -1404,7 +668,7 @@
  * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files,
    editor swap files, etc.
  * #81: Document user-defined hooks run before/after backup, or on error.
- * Add code style guidelines to the documentation.
+ * Add code style guidelines to the documention.
 
 1.2.0
  * #61: Support for Borg --list option via borgmatic command-line to list all archives.

README.md

@@ -11,86 +11,68 @@ borgmatic is simple, configuration-driven backup software for servers and
 workstations. Protect your files with client-side encryption. Backup your
 databases too. Monitor it all with integrated third-party services.
 
-The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic/</a>
+The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic</a>.
 
 Here's an example configuration file:
 
 ```yaml
-# List of source directories to backup.
-source_directories:
-    - /home
-    - /etc
+location:
+    # List of source directories to backup.
+    source_directories:
+        - /home
+        - /etc
 
-# Paths of local or remote repositories to backup to.
-repositories:
-    - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo
-      label: borgbase
-    - path: /var/lib/backups/local.borg
-      label: local
+    # Paths of local or remote repositories to backup to.
+    repositories:
+        - 1234@usw-s001.rsync.net:backups.borg
+        - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
+        - /var/lib/backups/local.borg
 
-# Retention policy for how many backups to keep.
-keep_daily: 7
-keep_weekly: 4
-keep_monthly: 6
+retention:
+    # Retention policy for how many backups to keep.
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6
 
-# List of checks to run to validate your backups.
-checks:
-    - name: repository
-    - name: archives
-      frequency: 2 weeks
+consistency:
+    # List of checks to run to validate your backups.
+    checks:
+        - name: repository
+        - name: archives
+          frequency: 2 weeks
 
-# Custom preparation scripts to run.
-before_backup:
-    - prepare-for-backup.sh
+hooks:
+    # Custom preparation scripts to run.
+    before_backup:
+        - prepare-for-backup.sh
 
-# Databases to dump and include in backups.
-postgresql_databases:
-    - name: users
+    # Databases to dump and include in backups.
+    postgresql_databases:
+        - name: users
 
-# Third-party services to notify you if backups aren't happening.
-healthchecks:
-    ping_url: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
+    # Third-party services to notify you if backups aren't happening.
+    healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
 ```
 
+Want to see borgmatic in action? Check out the <a
+href="https://asciinema.org/a/203761?autoplay=1" target="_blank">screencast</a>.
+
+<a href="https://asciinema.org/a/203761?autoplay=1" target="_blank"><img src="https://asciinema.org/a/203761.png" width="480"></a>
+
 borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 
 ## Integrations
 
-### Data
-
-<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://openzfs.org/"><img src="docs/static/openzfs.png" alt="OpenZFS" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://btrfs.readthedocs.io/"><img src="docs/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sourceware.org/lvm2/"><img src="docs/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://rclone.org"><img src="docs/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-
-
-### Monitoring
-
-<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://uptime.kuma.pet/"><img src="docs/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.pushover.net/"><img src="docs/static/pushover.png" alt="Pushover" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://grafana.com/oss/loki/"><img src="docs/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.zabbix.com/"><img src="docs/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sentry.io/"><img src="docs/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-
-
-### Credentials
-
-<a href="https://systemd.io/"><img src="docs/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.docker.com/"><img src="docs/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://podman.io/"><img src="docs/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://keepassxc.org/"><img src="docs/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 
 
 ## Getting started
@@ -98,8 +80,8 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 Your first step is to [install and configure
 borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/).
 
-For additional documentation, check out the links above (left panel on wide screens)
-for <a href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
+For additional documentation, check out the links above for <a
+href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
 reference guides</a>.
 
 
@@ -107,49 +89,38 @@ reference guides</a>.
 
 Need somewhere to store your encrypted off-site backups? The following hosting
 providers include specific support for Borg/borgmatic—and fund borgmatic
-development and hosting when you use these referral links to sign up:
+development and hosting when you use these links to sign up. (These are
+referral links, but without any tracking scripts or cookies.)
 
 <ul>
  <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
- <li class="referral"><a href="https://hetzner.cloud/?ref=v9dOJ98Ic9I8">Hetzner</a>: A "storage box" that includes support for Borg</li>
 </ul>
 
-Additionally, rsync.net has a compatible storage offering, but does not fund
-borgmatic development or hosting.
+Additionally, [rsync.net](https://www.rsync.net/products/borg.html) and
+[Hetzner](https://www.hetzner.com/storage/storage-box) have compatible storage
+offerings, but do not currently fund borgmatic development or hosting.
 
 ## Support and contributing
 
 ### Issues
 
-Are you experiencing an issue with borgmatic? Or do you have an idea for a
-feature enhancement? Head on over to our [issue
-tracker](https://projects.torsion.org/borgmatic-collective/borgmatic/issues).
-In order to create a new issue or add a comment, you'll need to
-[register](https://projects.torsion.org/user/sign_up?invite_code=borgmatic)
-first. If you prefer to use an existing GitHub account, you can skip account
-creation and [login directly](https://projects.torsion.org/user/login).
+You've got issues? Or an idea for a feature enhancement? We've got an [issue
+tracker](https://projects.torsion.org/borgmatic-collective/borgmatic/issues). In order to
+create a new issue or comment on an issue, you'll need to [login
+first](https://projects.torsion.org/user/login). Note that you can login with
+an existing GitHub account if you prefer.
+
+If you'd like to chat with borgmatic developers or users, head on over to the
+`#borgmatic` IRC channel on Libera Chat, either via <a
+href="https://web.libera.chat/#borgmatic">web chat</a> or a
+native <a href="ircs://irc.libera.chat:6697">IRC client</a>. If you
+don't get a response right away, please hang around a while—or file a ticket
+instead.
 
 Also see the [security
 policy](https://torsion.org/borgmatic/docs/security-policy/) for any security
 issues.
 
-
-### Social
-
-Follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic).
-
-
-### Chat
-
-To chat with borgmatic developers or users, check out the `#borgmatic`
-IRC channel on Libera Chat, either via <a
-href="https://web.libera.chat/#borgmatic">web chat</a> or a native <a
-href="ircs://irc.libera.chat:6697">IRC client</a>. If you don't get a response
-right away, please hang around a while—or file a ticket instead.
-
-
-### Other
-
 Other questions or comments? Contact
 [witten@torsion.org](mailto:witten@torsion.org).
 
@@ -164,23 +135,14 @@ borgmatic is licensed under the GNU General Public License version 3 or any
 later version.
 
 If you'd like to contribute to borgmatic development, please feel free to
-submit a [Pull
-Request](https://projects.torsion.org/borgmatic-collective/borgmatic/pulls) or
-open an
-[issue](https://projects.torsion.org/borgmatic-collective/borgmatic/issues) to
-discuss your idea. Note that you'll need to
-[register](https://projects.torsion.org/user/sign_up?invite_code=borgmatic)
-first. We also accept Pull Requests on GitHub, if that's more your thing. In
-general, contributions are very welcome. We don't bite!
+submit a [Pull Request](https://projects.torsion.org/borgmatic-collective/borgmatic/pulls)
+or open an [issue](https://projects.torsion.org/borgmatic-collective/borgmatic/issues) first
+to discuss your idea. We also accept Pull Requests on GitHub, if that's more
+your thing. In general, contributions are very welcome. We don't bite! 
 
 Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
 
-### Recent contributors
+<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/master)</a>
 
-Thanks to all borgmatic contributors! There are multiple ways to contribute to
-this project, so the following includes those who have fixed bugs, contributed
-features, *or* filed tickets.
-
-{% include borgmatic/contributors.html %}

SECURITY.md

@@ -7,8 +7,8 @@ permalink: security-policy/index.html
 
 While we want to hear about security vulnerabilities in all versions of
 borgmatic, security fixes are only made to the most recently released version.
-It's not practical for our small volunteer effort to maintain multiple release
-branches and put out separate security patches for each.
+It's simply not practical for our small volunteer effort to maintain multiple
+release branches and put out separate security patches for each.
 
 ## Reporting a vulnerability
 

borgmatic/actions/arguments.py

@@ -1,9 +0,0 @@
-import argparse
-
-
-def update_arguments(arguments, **updates):
-    '''
-    Given an argparse.Namespace instance of command-line arguments and one or more keyword argument
-    updates to perform, return a copy of the arguments with those updates applied.
-    '''
-    return argparse.Namespace(**dict(vars(arguments), **updates))

borgmatic/actions/borg.py

@@ -1,43 +0,0 @@
-import logging
-
-import borgmatic.borg.borg
-import borgmatic.borg.repo_list
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_borg(
-    repository,
-    config,
-    local_borg_version,
-    borg_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "borg" action for the given repository.
-    '''
-    if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, borg_arguments.repository
-    ):
-        logger.info('Running arbitrary Borg command')
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            repository['path'],
-            borg_arguments.archive,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        borgmatic.borg.borg.run_arbitrary_borg(
-            repository['path'],
-            config,
-            local_borg_version,
-            options=borg_arguments.options,
-            archive=archive_name,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/break_lock.py

@@ -1,32 +0,0 @@
-import logging
-
-import borgmatic.borg.break_lock
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_break_lock(
-    repository,
-    config,
-    local_borg_version,
-    break_lock_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "break-lock" action for the given repository.
-    '''
-    if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, break_lock_arguments.repository
-    ):
-        logger.info('Breaking repository and cache locks')
-        borgmatic.borg.break_lock.break_lock(
-            repository['path'],
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/change_passphrase.py

@@ -1,36 +0,0 @@
-import logging
-
-import borgmatic.borg.change_passphrase
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_change_passphrase(
-    repository,
-    config,
-    local_borg_version,
-    change_passphrase_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "key change-passphrase" action for the given repository.
-    '''
-    if (
-        change_passphrase_arguments.repository is None
-        or borgmatic.config.validate.repositories_match(
-            repository, change_passphrase_arguments.repository
-        )
-    ):
-        logger.info('Changing repository passphrase')
-        borgmatic.borg.change_passphrase.change_passphrase(
-            repository['path'],
-            config,
-            local_borg_version,
-            change_passphrase_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/check.py

@@ -1,764 +0,0 @@
-import calendar
-import datetime
-import hashlib
-import itertools
-import logging
-import os
-import pathlib
-import random
-import shutil
-
-import borgmatic.actions.create
-import borgmatic.borg.check
-import borgmatic.borg.create
-import borgmatic.borg.environment
-import borgmatic.borg.extract
-import borgmatic.borg.list
-import borgmatic.borg.repo_list
-import borgmatic.borg.state
-import borgmatic.config.paths
-import borgmatic.config.validate
-import borgmatic.execute
-import borgmatic.hooks.command
-
-DEFAULT_CHECKS = (
-    {'name': 'repository', 'frequency': '1 month'},
-    {'name': 'archives', 'frequency': '1 month'},
-)
-
-
-logger = logging.getLogger(__name__)
-
-
-def parse_checks(config, only_checks=None):
-    '''
-    Given a configuration dict with a "checks" sequence of dicts and an optional list of override
-    checks, return a tuple of named checks to run.
-
-    For example, given a config of:
-
-        {'checks': ({'name': 'repository'}, {'name': 'archives'})}
-
-    This will be returned as:
-
-        ('repository', 'archives')
-
-    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If a checks value
-    has a name of "disabled", return an empty tuple, meaning that no checks should be run.
-    '''
-    checks = only_checks or tuple(
-        check_config['name'] for check_config in (config.get('checks', None) or DEFAULT_CHECKS)
-    )
-    checks = tuple(check.lower() for check in checks)
-
-    if 'disabled' in checks:
-        logger.warning(
-            'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead'
-        )
-        if len(checks) > 1:
-            logger.warning(
-                'Multiple checks are configured, but one of them is "disabled"; not running any checks'
-            )
-        return ()
-
-    return checks
-
-
-def parse_frequency(frequency):
-    '''
-    Given a frequency string with a number and a unit of time, return a corresponding
-    datetime.timedelta instance or None if the frequency is None or "always".
-
-    For instance, given "3 weeks", return datetime.timedelta(weeks=3)
-
-    Raise ValueError if the given frequency cannot be parsed.
-    '''
-    if not frequency:
-        return None
-
-    frequency = frequency.strip().lower()
-
-    if frequency == 'always':
-        return None
-
-    try:
-        number, time_unit = frequency.split(' ')
-        number = int(number)
-    except ValueError:
-        raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
-
-    if not time_unit.endswith('s'):
-        time_unit += 's'
-
-    if time_unit == 'months':
-        number *= 30
-        time_unit = 'days'
-    elif time_unit == 'years':
-        number *= 365
-        time_unit = 'days'
-
-    try:
-        return datetime.timedelta(**{time_unit: number})
-    except TypeError:
-        raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
-
-
-WEEKDAY_DAYS = calendar.day_name[0:5]
-WEEKEND_DAYS = calendar.day_name[5:7]
-
-
-def filter_checks_on_frequency(
-    config,
-    borg_repository_id,
-    checks,
-    force,
-    archives_check_id=None,
-    datetime_now=datetime.datetime.now,
-):
-    '''
-    Given a configuration dict with a "checks" sequence of dicts, a Borg repository ID, a sequence
-    of checks, whether to force checks to run, and an ID for the archives check potentially being
-    run (if any), filter down those checks based on the configured "frequency" for each check as
-    compared to its check time file.
-
-    In other words, a check whose check time file's timestamp is too new (based on the configured
-    frequency) will get cut from the returned sequence of checks. Example:
-
-    config = {
-        'checks': [
-            {
-                'name': 'archives',
-                'frequency': '2 weeks',
-            },
-        ]
-    }
-
-    When this function is called with that config and "archives" in checks, "archives" will get
-    filtered out of the returned result if its check time file is newer than 2 weeks old, indicating
-    that it's not yet time to run that check again.
-
-    Raise ValueError if a frequency cannot be parsed.
-    '''
-    if not checks:
-        return checks
-
-    filtered_checks = list(checks)
-
-    if force:
-        return tuple(filtered_checks)
-
-    for check_config in config.get('checks', DEFAULT_CHECKS):
-        check = check_config['name']
-        if checks and check not in checks:
-            continue
-
-        only_run_on = check_config.get('only_run_on')
-        if only_run_on:
-            # Use a dict instead of a set to preserve ordering.
-            days = dict.fromkeys(only_run_on)
-
-            if 'weekday' in days:
-                days = {
-                    **dict.fromkeys(day for day in days if day != 'weekday'),
-                    **dict.fromkeys(WEEKDAY_DAYS),
-                }
-            if 'weekend' in days:
-                days = {
-                    **dict.fromkeys(day for day in days if day != 'weekend'),
-                    **dict.fromkeys(WEEKEND_DAYS),
-                }
-
-            if calendar.day_name[datetime_now().weekday()] not in days:
-                logger.info(
-                    f"Skipping {check} check due to day of the week; check only runs on {'/'.join(days)} (use --force to check anyway)"
-                )
-                filtered_checks.remove(check)
-                continue
-
-        frequency_delta = parse_frequency(check_config.get('frequency'))
-        if not frequency_delta:
-            continue
-
-        check_time = probe_for_check_time(config, borg_repository_id, check, archives_check_id)
-        if not check_time:
-            continue
-
-        # If we've not yet reached the time when the frequency dictates we're ready for another
-        # check, skip this check.
-        if datetime_now() < check_time + frequency_delta:
-            remaining = check_time + frequency_delta - datetime_now()
-            logger.info(
-                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
-            )
-            filtered_checks.remove(check)
-
-    return tuple(filtered_checks)
-
-
-def make_archives_check_id(archive_filter_flags):
-    '''
-    Given a sequence of flags to filter archives, return a unique hash corresponding to those
-    particular flags. If there are no flags, return None.
-    '''
-    if not archive_filter_flags:
-        return None
-
-    return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest()
-
-
-def make_check_time_path(config, borg_repository_id, check_type, archives_check_id=None):
-    '''
-    Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
-    "archives", etc.), and a unique hash of the archives filter flags, return a path for recording
-    that check's time (the time of that check last occurring).
-    '''
-    borgmatic_state_directory = borgmatic.config.paths.get_borgmatic_state_directory(config)
-
-    if check_type in ('archives', 'data'):
-        return os.path.join(
-            borgmatic_state_directory,
-            'checks',
-            borg_repository_id,
-            check_type,
-            archives_check_id if archives_check_id else 'all',
-        )
-
-    return os.path.join(
-        borgmatic_state_directory,
-        'checks',
-        borg_repository_id,
-        check_type,
-    )
-
-
-def write_check_time(path):  # pragma: no cover
-    '''
-    Record a check time of now as the modification time of the given path.
-    '''
-    logger.debug(f'Writing check time at {path}')
-
-    os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
-    pathlib.Path(path, mode=0o600).touch()
-
-
-def read_check_time(path):
-    '''
-    Return the check time based on the modification time of the given path. Return None if the path
-    doesn't exist.
-    '''
-    logger.debug(f'Reading check time from {path}')
-
-    try:
-        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
-    except FileNotFoundError:
-        return None
-
-
-def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
-    '''
-    Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
-    "archives", etc.), and a unique hash of the archives filter flags, return the corresponding
-    check time or None if such a check time does not exist.
-
-    When the check type is "archives" or "data", this function probes two different paths to find
-    the check time, e.g.:
-
-      ~/.borgmatic/checks/1234567890/archives/9876543210
-      ~/.borgmatic/checks/1234567890/archives/all
-
-    ... and returns the maximum modification time of the files found (if any). The first path
-    represents a more specific archives check time (a check on a subset of archives), and the second
-    is a fallback to the last "all" archives check.
-
-    For other check types, this function reads from a single check time path, e.g.:
-
-      ~/.borgmatic/checks/1234567890/repository
-    '''
-    check_times = (
-        read_check_time(group[0])
-        for group in itertools.groupby(
-            (
-                make_check_time_path(config, borg_repository_id, check, archives_check_id),
-                make_check_time_path(config, borg_repository_id, check),
-            )
-        )
-    )
-
-    try:
-        return max(check_time for check_time in check_times if check_time)
-    except ValueError:
-        return None
-
-
-def upgrade_check_times(config, borg_repository_id):
-    '''
-    Given a configuration dict and a Borg repository ID, upgrade any corresponding check times on
-    disk from old-style paths to new-style paths.
-
-    One upgrade performed is moving the checks directory from:
-
-      {borgmatic_source_directory}/checks (e.g., ~/.borgmatic/checks)
-
-    to:
-
-      {borgmatic_state_directory}/checks (e.g. ~/.local/state/borgmatic)
-
-    Another upgrade is renaming an archive or data check path that looks like:
-
-      {borgmatic_state_directory}/checks/1234567890/archives
-
-    to:
-
-      {borgmatic_state_directory}/checks/1234567890/archives/all
-    '''
-    borgmatic_source_checks_path = os.path.join(
-        borgmatic.config.paths.get_borgmatic_source_directory(config), 'checks'
-    )
-    borgmatic_state_path = borgmatic.config.paths.get_borgmatic_state_directory(config)
-    borgmatic_state_checks_path = os.path.join(borgmatic_state_path, 'checks')
-
-    if os.path.exists(borgmatic_source_checks_path) and not os.path.exists(
-        borgmatic_state_checks_path
-    ):
-        logger.debug(
-            f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}'
-        )
-        os.makedirs(borgmatic_state_path, mode=0o700, exist_ok=True)
-        shutil.move(borgmatic_source_checks_path, borgmatic_state_checks_path)
-
-    for check_type in ('archives', 'data'):
-        new_path = make_check_time_path(config, borg_repository_id, check_type, 'all')
-        old_path = os.path.dirname(new_path)
-        temporary_path = f'{old_path}.temp'
-
-        if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
-            continue
-
-        logger.debug(f'Upgrading archives check time file from {old_path} to {new_path}')
-
-        try:
-            shutil.move(old_path, temporary_path)
-        except FileNotFoundError:
-            pass
-
-        os.mkdir(old_path)
-        shutil.move(temporary_path, new_path)
-
-
-def collect_spot_check_source_paths(
-    repository,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path,
-    remote_path,
-    borgmatic_runtime_directory,
-):
-    '''
-    Given a repository configuration dict, a configuration dict, the local Borg version, global
-    arguments as an argparse.Namespace instance, the local Borg path, and the remote Borg path,
-    collect the source paths that Borg would use in an actual create (but only include files).
-    '''
-    stream_processes = any(
-        borgmatic.hooks.dispatch.call_hooks(
-            'use_streaming',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-        ).values()
-    )
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    (create_flags, create_positional_arguments, pattern_file) = (
-        borgmatic.borg.create.make_base_create_command(
-            dry_run=True,
-            repository_path=repository['path'],
-            config=config,
-            patterns=borgmatic.actions.create.process_patterns(
-                borgmatic.actions.create.collect_patterns(config),
-                working_directory,
-            ),
-            local_borg_version=local_borg_version,
-            global_arguments=global_arguments,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
-            local_path=local_path,
-            remote_path=remote_path,
-            list_files=True,
-            stream_processes=stream_processes,
-        )
-    )
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    paths_output = borgmatic.execute.execute_command_and_capture_output(
-        create_flags + create_positional_arguments,
-        capture_stderr=True,
-        environment=borgmatic.borg.environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )
-
-    paths = tuple(
-        path_line.split(' ', 1)[1]
-        for path_line in paths_output.splitlines()
-        if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
-    )
-
-    return tuple(
-        path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
-    )
-
-
-BORG_DIRECTORY_FILE_TYPE = 'd'
-BORG_PIPE_FILE_TYPE = 'p'
-
-
-def collect_spot_check_archive_paths(
-    repository,
-    archive,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path,
-    remote_path,
-    borgmatic_runtime_directory,
-):
-    '''
-    Given a repository configuration dict, the name of the latest archive, a configuration dict, the
-    local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the
-    remote Borg path, and the borgmatic runtime directory, collect the paths from the given archive
-    (but only include files and symlinks and exclude borgmatic runtime directories).
-
-    These paths do not have a leading slash, as that's how Borg stores them. As a result, we don't
-    know whether they came from absolute or relative source directories.
-    '''
-    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
-
-    return tuple(
-        path
-        for line in borgmatic.borg.list.capture_archive_listing(
-            repository['path'],
-            archive,
-            config,
-            local_borg_version,
-            global_arguments,
-            path_format='{type} {path}{NUL}',  # noqa: FS003
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        for (file_type, path) in (line.split(' ', 1),)
-        if file_type not in (BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE)
-        if pathlib.Path('borgmatic') not in pathlib.Path(path).parents
-        if pathlib.Path(borgmatic_source_directory.lstrip(os.path.sep))
-        not in pathlib.Path(path).parents
-        if pathlib.Path(borgmatic_runtime_directory.lstrip(os.path.sep))
-        not in pathlib.Path(path).parents
-    )
-
-
-SAMPLE_PATHS_SUBSET_COUNT = 10000
-
-
-def compare_spot_check_hashes(
-    repository,
-    archive,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path,
-    remote_path,
-    source_paths,
-):
-    '''
-    Given a repository configuration dict, the name of the latest archive, a configuration dict, the
-    local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the
-    remote Borg path, and spot check source paths, compare the hashes for a sampling of the source
-    paths with hashes from corresponding paths in the given archive. Return a sequence of the paths
-    that fail that hash comparison.
-    '''
-    # Based on the configured sample percentage, come up with a list of random sample files from the
-    # source directories.
-    spot_check_config = next(check for check in config['checks'] if check['name'] == 'spot')
-    sample_count = max(
-        int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
-    )
-    source_sample_paths = tuple(random.sample(source_paths, sample_count))
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-    existing_source_sample_paths = {
-        source_path
-        for source_path in source_sample_paths
-        if os.path.exists(os.path.join(working_directory or '', source_path))
-    }
-    logger.debug(
-        f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
-    )
-
-    source_sample_paths_iterator = iter(source_sample_paths)
-    source_hashes = {}
-    archive_hashes = {}
-
-    # Only hash a few thousand files at a time (a subset of the total paths) to avoid an "Argument
-    # list too long" shell error.
-    while True:
-        # Hash each file in the sample paths (if it exists).
-        source_sample_paths_subset = tuple(
-            itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT)
-        )
-        if not source_sample_paths_subset:
-            break
-
-        hash_output = borgmatic.execute.execute_command_and_capture_output(
-            (spot_check_config.get('xxh64sum_command', 'xxh64sum'),)
-            + tuple(
-                path for path in source_sample_paths_subset if path in existing_source_sample_paths
-            ),
-            working_directory=working_directory,
-        )
-
-        source_hashes.update(
-            **dict(
-                (reversed(line.split('  ', 1)) for line in hash_output.splitlines()),
-                # Represent non-existent files as having empty hashes so the comparison below still works.
-                **{
-                    path: ''
-                    for path in source_sample_paths_subset
-                    if path not in existing_source_sample_paths
-                },
-            )
-        )
-
-        # Get the hash for each file in the archive.
-        archive_hashes.update(
-            **dict(
-                reversed(line.split(' ', 1))
-                for line in borgmatic.borg.list.capture_archive_listing(
-                    repository['path'],
-                    archive,
-                    config,
-                    local_borg_version,
-                    global_arguments,
-                    list_paths=source_sample_paths_subset,
-                    path_format='{xxh64} {path}{NUL}',  # noqa: FS003
-                    local_path=local_path,
-                    remote_path=remote_path,
-                )
-                if line
-            )
-        )
-
-    # Compare the source hashes with the archive hashes to see how many match.
-    failing_paths = []
-
-    for path, source_hash in source_hashes.items():
-        archive_hash = archive_hashes.get(path.lstrip(os.path.sep))
-
-        if archive_hash is not None and archive_hash == source_hash:
-            continue
-
-        failing_paths.append(path)
-
-    return tuple(failing_paths)
-
-
-def spot_check(
-    repository,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path,
-    remote_path,
-    borgmatic_runtime_directory,
-):
-    '''
-    Given a repository dict, a loaded configuration dict, the local Borg version, global arguments
-    as an argparse.Namespace instance, the local Borg path, the remote Borg path, and the borgmatic
-    runtime directory, perform a spot check for the latest archive in the given repository.
-
-    A spot check compares file counts and also the hashes for a random sampling of source files on
-    disk to those stored in the latest archive. If any differences are beyond configured tolerances,
-    then the check fails.
-    '''
-    logger.debug('Running spot check')
-
-    try:
-        spot_check_config = next(
-            check for check in config.get('checks', ()) if check.get('name') == 'spot'
-        )
-    except StopIteration:
-        raise ValueError('Cannot run spot check because it is unconfigured')
-
-    if spot_check_config['data_tolerance_percentage'] > spot_check_config['data_sample_percentage']:
-        raise ValueError(
-            'The data_tolerance_percentage must be less than or equal to the data_sample_percentage'
-        )
-
-    source_paths = collect_spot_check_source_paths(
-        repository,
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path,
-        remote_path,
-        borgmatic_runtime_directory,
-    )
-    logger.debug(f'{len(source_paths)} total source paths for spot check')
-
-    archive = borgmatic.borg.repo_list.resolve_archive_name(
-        repository['path'],
-        'latest',
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path,
-        remote_path,
-    )
-    logger.debug(f'Using archive {archive} for spot check')
-
-    archive_paths = collect_spot_check_archive_paths(
-        repository,
-        archive,
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path,
-        remote_path,
-        borgmatic_runtime_directory,
-    )
-    logger.debug(f'{len(archive_paths)} total archive paths for spot check')
-
-    if len(source_paths) == 0:
-        logger.debug(
-            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths)) or "none"}'
-        )
-        raise ValueError(
-            'Spot check failed: There are no source paths to compare against the archive'
-        )
-
-    # Calculate the percentage delta between the source paths count and the archive paths count, and
-    # compare that delta to the configured count tolerance percentage.
-    count_delta_percentage = abs(len(source_paths) - len(archive_paths)) / len(source_paths) * 100
-
-    if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
-        rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths)
-        logger.debug(
-            f'Paths in source paths but not latest archive: {", ".join(rootless_source_paths - set(archive_paths)) or "none"}'
-        )
-        logger.debug(
-            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths) - rootless_source_paths) or "none"}'
-        )
-        raise ValueError(
-            f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'
-        )
-
-    failing_paths = compare_spot_check_hashes(
-        repository,
-        archive,
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path,
-        remote_path,
-        source_paths,
-    )
-
-    # Error if the percentage of failing hashes exceeds the configured tolerance percentage.
-    logger.debug(f'{len(failing_paths)} non-matching spot check hashes')
-    data_tolerance_percentage = spot_check_config['data_tolerance_percentage']
-    failing_percentage = (len(failing_paths) / len(source_paths)) * 100
-
-    if failing_percentage > data_tolerance_percentage:
-        logger.debug(
-            f'Source paths with data not matching the latest archive: {", ".join(failing_paths)}'
-        )
-        raise ValueError(
-            f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)'
-        )
-
-    logger.info(
-        f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta'
-    )
-
-
-def run_check(
-    config_filename,
-    repository,
-    config,
-    local_borg_version,
-    check_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "check" action for the given repository.
-
-    Raise ValueError if the Borg repository ID cannot be determined.
-    '''
-    if check_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, check_arguments.repository
-    ):
-        return
-
-    logger.info('Running consistency checks')
-
-    repository_id = borgmatic.borg.check.get_repository_id(
-        repository['path'],
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path=local_path,
-        remote_path=remote_path,
-    )
-    upgrade_check_times(config, repository_id)
-    configured_checks = parse_checks(config, check_arguments.only_checks)
-    archive_filter_flags = borgmatic.borg.check.make_archive_filter_flags(
-        local_borg_version, config, configured_checks, check_arguments
-    )
-    archives_check_id = make_archives_check_id(archive_filter_flags)
-    checks = filter_checks_on_frequency(
-        config,
-        repository_id,
-        configured_checks,
-        check_arguments.force,
-        archives_check_id,
-    )
-    borg_specific_checks = set(checks).intersection({'repository', 'archives', 'data'})
-
-    if borg_specific_checks:
-        borgmatic.borg.check.check_archives(
-            repository['path'],
-            config,
-            local_borg_version,
-            check_arguments,
-            global_arguments,
-            borg_specific_checks,
-            archive_filter_flags,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        for check in borg_specific_checks:
-            write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
-
-    if 'extract' in checks:
-        borgmatic.borg.extract.extract_last_archive_dry_run(
-            config,
-            local_borg_version,
-            global_arguments,
-            repository['path'],
-            config.get('lock_wait'),
-            local_path,
-            remote_path,
-        )
-        write_check_time(make_check_time_path(config, repository_id, 'extract'))
-
-    if 'spot' in checks:
-        with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-            spot_check(
-                repository,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-                borgmatic_runtime_directory,
-            )
-        write_check_time(make_check_time_path(config, repository_id, 'spot'))

borgmatic/actions/compact.py

@@ -1,45 +0,0 @@
-import logging
-
-import borgmatic.borg.compact
-import borgmatic.borg.feature
-import borgmatic.config.validate
-import borgmatic.hooks.command
-
-logger = logging.getLogger(__name__)
-
-
-def run_compact(
-    config_filename,
-    repository,
-    config,
-    local_borg_version,
-    compact_arguments,
-    global_arguments,
-    dry_run_label,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "compact" action for the given repository.
-    '''
-    if compact_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, compact_arguments.repository
-    ):
-        return
-
-    if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
-        logger.info(f'Compacting segments{dry_run_label}')
-        borgmatic.borg.compact.compact_segments(
-            global_arguments.dry_run,
-            repository['path'],
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            progress=compact_arguments.progress,
-            cleanup_commits=compact_arguments.cleanup_commits,
-            threshold=compact_arguments.threshold,
-        )
-    else:  # pragma: nocover
-        logger.info('Skipping compact (only available/needed in Borg 1.2+)')

borgmatic/actions/config/bootstrap.py

@@ -1,131 +0,0 @@
-import json
-import logging
-import os
-
-import borgmatic.borg.extract
-import borgmatic.borg.repo_list
-import borgmatic.config.paths
-import borgmatic.config.validate
-import borgmatic.hooks.command
-
-logger = logging.getLogger(__name__)
-
-
-def make_bootstrap_config(bootstrap_arguments):
-    '''
-    Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict.
-    '''
-    return {
-        'ssh_command': bootstrap_arguments.ssh_command,
-        # In case the repo has been moved or is accessed from a different path at the point of
-        # bootstrapping.
-        'relocated_repo_access_is_ok': True,
-    }
-
-
-def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version):
-    '''
-    Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the
-    repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory,
-    borgmatic source directory, destination directory, and whether to strip components), the global
-    arguments as an argparse.Namespace (containing the dry run flag and the local borg version),
-    return the config paths from the manifest.json file in the borgmatic source directory or runtime
-    directory after extracting it from the repository archive.
-
-    Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
-    expected configuration path data.
-    '''
-    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(
-        {'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory}
-    )
-    config = make_bootstrap_config(bootstrap_arguments)
-
-    # Probe for the manifest file in multiple locations, as the default location has moved to the
-    # borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But we
-    # still want to support reading the manifest from previously created archives as well.
-    with borgmatic.config.paths.Runtime_directory(
-        {'user_runtime_directory': bootstrap_arguments.user_runtime_directory},
-    ) as borgmatic_runtime_directory:
-        for base_directory in (
-            'borgmatic',
-            borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
-            borgmatic_source_directory,
-        ):
-            borgmatic_manifest_path = 'sh:' + os.path.join(
-                base_directory, 'bootstrap', 'manifest.json'
-            )
-
-            extract_process = borgmatic.borg.extract.extract_archive(
-                global_arguments.dry_run,
-                bootstrap_arguments.repository,
-                archive_name,
-                [borgmatic_manifest_path],
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path=bootstrap_arguments.local_path,
-                remote_path=bootstrap_arguments.remote_path,
-                extract_to_stdout=True,
-            )
-            manifest_json = extract_process.stdout.read()
-
-            if manifest_json:
-                break
-        else:
-            raise ValueError(
-                'Cannot read configuration paths from archive due to missing bootstrap manifest'
-            )
-
-    try:
-        manifest_data = json.loads(manifest_json)
-    except json.JSONDecodeError as error:
-        raise ValueError(
-            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}'
-        )
-
-    try:
-        return manifest_data['config_paths']
-    except KeyError:
-        raise ValueError(
-            'Cannot read configuration paths from archive due to invalid bootstrap manifest'
-        )
-
-
-def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
-    '''
-    Run the "bootstrap" action for the given repository.
-
-    Raise ValueError if the bootstrap configuration could not be loaded.
-    Raise CalledProcessError or OSError if Borg could not be run.
-    '''
-    config = make_bootstrap_config(bootstrap_arguments)
-    archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-        bootstrap_arguments.repository,
-        bootstrap_arguments.archive,
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path=bootstrap_arguments.local_path,
-        remote_path=bootstrap_arguments.remote_path,
-    )
-    manifest_config_paths = get_config_paths(
-        archive_name, bootstrap_arguments, global_arguments, local_borg_version
-    )
-
-    logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
-
-    borgmatic.borg.extract.extract_archive(
-        global_arguments.dry_run,
-        bootstrap_arguments.repository,
-        archive_name,
-        [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path=bootstrap_arguments.local_path,
-        remote_path=bootstrap_arguments.remote_path,
-        extract_to_stdout=False,
-        destination_path=bootstrap_arguments.destination,
-        strip_components=bootstrap_arguments.strip_components,
-        progress=bootstrap_arguments.progress,
-    )

borgmatic/actions/config/generate.py

@@ -1,48 +0,0 @@
-import logging
-
-import borgmatic.config.generate
-import borgmatic.config.validate
-import borgmatic.logger
-
-logger = logging.getLogger(__name__)
-
-
-def run_generate(generate_arguments, global_arguments):
-    '''
-    Given the generate arguments and the global arguments, each as an argparse.Namespace instance,
-    run the "generate" action.
-
-    Raise FileExistsError if a file already exists at the destination path and the generate
-    arguments do not have overwrite set.
-    '''
-    borgmatic.logger.add_custom_log_levels()
-    dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
-
-    logger.answer(
-        f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
-    )
-
-    borgmatic.config.generate.generate_sample_configuration(
-        global_arguments.dry_run,
-        generate_arguments.source_filename,
-        generate_arguments.destination_filename,
-        borgmatic.config.validate.schema_filename(),
-        overwrite=generate_arguments.overwrite,
-    )
-
-    if generate_arguments.source_filename:
-        logger.answer(
-            f'''
-Merged in the contents of configuration file at: {generate_arguments.source_filename}
-To review the changes made, run:
-
-    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
-        )
-
-    logger.answer(
-        '''
-This includes all available configuration options with example values, the few
-required options as indicated. Please edit the file to suit your needs.
-
-If you ever need help: https://torsion.org/borgmatic/#issues'''
-    )

borgmatic/actions/config/validate.py

@@ -1,25 +0,0 @@
-import logging
-
-import borgmatic.config.generate
-import borgmatic.logger
-
-logger = logging.getLogger(__name__)
-
-
-def run_validate(validate_arguments, configs):
-    '''
-    Given the validate arguments as an argparse.Namespace instance and a dict of configuration
-    filename to corresponding parsed configuration, run the "validate" action.
-
-    Most of the validation is actually performed implicitly by the standard borgmatic configuration
-    loading machinery prior to here, so this function mainly exists to support additional validate
-    flags like "--show".
-    '''
-    borgmatic.logger.add_custom_log_levels()
-
-    if validate_arguments.show:
-        for config_path, config in configs.items():
-            if len(configs) > 1:
-                logger.answer('---')
-
-            logger.answer(borgmatic.config.generate.render_configuration(config))

borgmatic/actions/create.py

@@ -1,346 +0,0 @@
-import glob
-import itertools
-import logging
-import os
-import pathlib
-
-import borgmatic.actions.json
-import borgmatic.borg.create
-import borgmatic.borg.pattern
-import borgmatic.config.paths
-import borgmatic.config.validate
-import borgmatic.hooks.command
-import borgmatic.hooks.dispatch
-
-logger = logging.getLogger(__name__)
-
-
-def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_style.NONE):
-    '''
-    Given a Borg pattern as a string, parse it into a borgmatic.borg.pattern.Pattern instance and
-    return it.
-    '''
-    try:
-        (pattern_type, remainder) = pattern_line.split(' ', maxsplit=1)
-    except ValueError:
-        raise ValueError(f'Invalid pattern: {pattern_line}')
-
-    try:
-        (parsed_pattern_style, path) = remainder.split(':', maxsplit=1)
-        pattern_style = borgmatic.borg.pattern.Pattern_style(parsed_pattern_style)
-    except ValueError:
-        pattern_style = default_style
-        path = remainder
-
-    return borgmatic.borg.pattern.Pattern(
-        path,
-        borgmatic.borg.pattern.Pattern_type(pattern_type),
-        borgmatic.borg.pattern.Pattern_style(pattern_style),
-        source=borgmatic.borg.pattern.Pattern_source.CONFIG,
-    )
-
-
-def collect_patterns(config):
-    '''
-    Given a configuration dict, produce a single sequence of patterns comprised of the configured
-    source directories, patterns, excludes, pattern files, and exclude files.
-
-    The idea is that Borg has all these different ways of specifying includes, excludes, source
-    directories, etc., but we'd like to collapse them all down to one common format (patterns) for
-    ease of manipulation within borgmatic.
-    '''
-    try:
-        return (
-            tuple(
-                borgmatic.borg.pattern.Pattern(
-                    source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
-                )
-                for source_directory in config.get('source_directories', ())
-            )
-            + tuple(
-                parse_pattern(pattern_line.strip())
-                for pattern_line in config.get('patterns', ())
-                if not pattern_line.lstrip().startswith('#')
-                if pattern_line.strip()
-            )
-            + tuple(
-                parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
-                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                )
-                for exclude_line in config.get('exclude_patterns', ())
-            )
-            + tuple(
-                parse_pattern(pattern_line.strip())
-                for filename in config.get('patterns_from', ())
-                for pattern_line in open(filename).readlines()
-                if not pattern_line.lstrip().startswith('#')
-                if pattern_line.strip()
-            )
-            + tuple(
-                parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
-                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                )
-                for filename in config.get('exclude_from', ())
-                for exclude_line in open(filename).readlines()
-                if not exclude_line.lstrip().startswith('#')
-                if exclude_line.strip()
-            )
-        )
-    except (FileNotFoundError, OSError) as error:
-        logger.debug(error)
-
-        raise ValueError(f'Cannot read patterns_from/exclude_from file: {error.filename}')
-
-
-def expand_directory(directory, working_directory):
-    '''
-    Given a directory path, expand any tilde (representing a user's home directory) and any globs
-    therein. Return a list of one or more resulting paths.
-
-    Take into account the given working directory so that relative paths are supported.
-    '''
-    expanded_directory = os.path.expanduser(directory)
-
-    # This would be a lot easier to do with glob(..., root_dir=working_directory), but root_dir is
-    # only available in Python 3.10+.
-    normalized_directory = os.path.join(working_directory or '', expanded_directory)
-    glob_paths = glob.glob(normalized_directory)
-
-    if not glob_paths:
-        return [expanded_directory]
-
-    working_directory_prefix = os.path.join(working_directory or '', '')
-
-    return [
-        (
-            glob_path
-            # If these are equal, that means we didn't add any working directory prefix above.
-            if normalized_directory == expanded_directory
-            # Remove the working directory prefix that we added above in order to make glob() work.
-            # We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
-            # hack.
-            else glob_path.removeprefix(working_directory_prefix)
-        )
-        for glob_path in glob_paths
-    ]
-
-
-def expand_patterns(patterns, working_directory=None, skip_paths=None):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
-    expand tildes and globs in each root pattern and expand just tildes in each non-root pattern.
-    The idea is that non-root patterns may be regular expressions or other pattern styles containing
-    "*" that borgmatic should not expand as a shell glob.
-
-    Return all the resulting patterns as a tuple.
-
-    If a set of paths are given to skip, then don't expand any patterns matching them.
-    '''
-    if patterns is None:
-        return ()
-
-    return tuple(
-        itertools.chain.from_iterable(
-            (
-                (
-                    borgmatic.borg.pattern.Pattern(
-                        expanded_path,
-                        pattern.type,
-                        pattern.style,
-                        pattern.device,
-                        pattern.source,
-                    )
-                    for expanded_path in expand_directory(pattern.path, working_directory)
-                )
-                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-                and pattern.path not in (skip_paths or ())
-                else (
-                    borgmatic.borg.pattern.Pattern(
-                        os.path.expanduser(pattern.path),
-                        pattern.type,
-                        pattern.style,
-                        pattern.device,
-                        pattern.source,
-                    ),
-                )
-            )
-            for pattern in patterns
-        )
-    )
-
-
-def device_map_patterns(patterns, working_directory=None):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
-    determine the identifier for the device on which the pattern's path resides—or None if the path
-    doesn't exist or is from a non-root pattern. Return an updated sequence of patterns with the
-    device field populated. But if the device field is already set, don't bother setting it again.
-
-    This is handy for determining whether two different pattern paths are on the same filesystem
-    (have the same device identifier).
-    '''
-    return tuple(
-        borgmatic.borg.pattern.Pattern(
-            pattern.path,
-            pattern.type,
-            pattern.style,
-            device=pattern.device
-            or (
-                os.stat(full_path).st_dev
-                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-                and os.path.exists(full_path)
-                else None
-            ),
-            source=pattern.source,
-        )
-        for pattern in patterns
-        for full_path in (os.path.join(working_directory or '', pattern.path),)
-    )
-
-
-def deduplicate_patterns(patterns):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
-    root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
-    "/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
-    modification.
-
-    The one exception to deduplication is two paths are on different filesystems (devices). In that
-    case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
-    one_file_system option is true).
-
-    The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
-    child patterns, because it will naturally spider the contents of the parent pattern's path. And
-    there are cases where Borg coming across the same file twice will result in duplicate reads and
-    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
-    Borg.
-    '''
-    deduplicated = {}  # Use just the keys as an ordered set.
-
-    for pattern in patterns:
-        if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
-            deduplicated[pattern] = True
-            continue
-
-        parents = pathlib.PurePath(pattern.path).parents
-
-        # If another directory in the given list is a parent of current directory (even n levels up)
-        # and both are on the same filesystem, then the current directory is a duplicate.
-        for other_pattern in patterns:
-            if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
-                continue
-
-            if any(
-                pathlib.PurePath(other_pattern.path) == parent
-                and pattern.device is not None
-                and other_pattern.device == pattern.device
-                for parent in parents
-            ):
-                break
-        else:
-            deduplicated[pattern] = True
-
-    return tuple(deduplicated.keys())
-
-
-def process_patterns(patterns, working_directory, skip_expand_paths=None):
-    '''
-    Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
-    "root" patterns, returning the resulting root and non-root patterns as a list.
-
-    If any paths are given to skip, don't expand them.
-    '''
-    skip_paths = set(skip_expand_paths or ())
-
-    return list(
-        deduplicate_patterns(
-            device_map_patterns(
-                expand_patterns(
-                    patterns,
-                    working_directory=working_directory,
-                    skip_paths=skip_paths,
-                )
-            )
-        )
-    )
-
-
-def run_create(
-    config_filename,
-    repository,
-    config,
-    config_paths,
-    local_borg_version,
-    create_arguments,
-    global_arguments,
-    dry_run_label,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "create" action for the given repository.
-
-    If create_arguments.json is True, yield the JSON output from creating the archive.
-    '''
-    if create_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, create_arguments.repository
-    ):
-        return
-
-    logger.info(f'Creating archive{dry_run_label}')
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )
-        patterns = process_patterns(collect_patterns(config), working_directory)
-        active_dumps = borgmatic.hooks.dispatch.call_hooks(
-            'dump_data_sources',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            config_paths,
-            borgmatic_runtime_directory,
-            patterns,
-            global_arguments.dry_run,
-        )
-
-        # Process the patterns again in case any data source hooks updated them. Without this step,
-        # we could end up with duplicate paths that cause Borg to hang when it tries to read from
-        # the same named pipe twice.
-        patterns = process_patterns(patterns, working_directory, skip_expand_paths=config_paths)
-        stream_processes = [process for processes in active_dumps.values() for process in processes]
-
-        json_output = borgmatic.borg.create.create_archive(
-            global_arguments.dry_run,
-            repository['path'],
-            config,
-            patterns,
-            local_borg_version,
-            global_arguments,
-            borgmatic_runtime_directory,
-            local_path=local_path,
-            remote_path=remote_path,
-            progress=create_arguments.progress,
-            stats=create_arguments.stats,
-            json=create_arguments.json,
-            list_files=create_arguments.list_files,
-            stream_processes=stream_processes,
-        )
-
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
-
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )

borgmatic/actions/delete.py

@@ -1,50 +0,0 @@
-import logging
-
-import borgmatic.actions.arguments
-import borgmatic.borg.delete
-import borgmatic.borg.repo_delete
-import borgmatic.borg.repo_list
-
-logger = logging.getLogger(__name__)
-
-
-def run_delete(
-    repository,
-    config,
-    local_borg_version,
-    delete_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "delete" action for the given repository and archive(s).
-    '''
-    if delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, delete_arguments.repository
-    ):
-        logger.answer('Deleting archives')
-
-        archive_name = (
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                delete_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            )
-            if delete_arguments.archive
-            else None
-        )
-
-        borgmatic.borg.delete.delete_archives(
-            repository,
-            config,
-            local_borg_version,
-            borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
-            global_arguments,
-            local_path,
-            remote_path,
-        )

borgmatic/actions/export_key.py

@@ -1,33 +0,0 @@
-import logging
-
-import borgmatic.borg.export_key
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_export_key(
-    repository,
-    config,
-    local_borg_version,
-    export_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "key export" action for the given repository.
-    '''
-    if export_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, export_arguments.repository
-    ):
-        logger.info('Exporting repository key')
-        borgmatic.borg.export_key.export_key(
-            repository['path'],
-            config,
-            local_borg_version,
-            export_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/export_tar.py

@@ -1,48 +0,0 @@
-import logging
-
-import borgmatic.borg.export_tar
-import borgmatic.borg.repo_list
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_export_tar(
-    repository,
-    config,
-    local_borg_version,
-    export_tar_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "export-tar" action for the given repository.
-    '''
-    if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, export_tar_arguments.repository
-    ):
-        logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file')
-        borgmatic.borg.export_tar.export_tar_archive(
-            global_arguments.dry_run,
-            repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                export_tar_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            export_tar_arguments.paths,
-            export_tar_arguments.destination,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            tar_filter=export_tar_arguments.tar_filter,
-            list_files=export_tar_arguments.list_files,
-            strip_components=export_tar_arguments.strip_components,
-        )

borgmatic/actions/extract.py

@@ -1,49 +0,0 @@
-import logging
-
-import borgmatic.borg.extract
-import borgmatic.borg.repo_list
-import borgmatic.config.validate
-import borgmatic.hooks.command
-
-logger = logging.getLogger(__name__)
-
-
-def run_extract(
-    config_filename,
-    repository,
-    config,
-    local_borg_version,
-    extract_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "extract" action for the given repository.
-    '''
-    if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, extract_arguments.repository
-    ):
-        logger.info(f'Extracting archive {extract_arguments.archive}')
-        borgmatic.borg.extract.extract_archive(
-            global_arguments.dry_run,
-            repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                extract_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            extract_arguments.paths,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            destination_path=extract_arguments.destination,
-            strip_components=extract_arguments.strip_components,
-            progress=extract_arguments.progress,
-        )

borgmatic/actions/import_key.py

@@ -1,33 +0,0 @@
-import logging
-
-import borgmatic.borg.import_key
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_import_key(
-    repository,
-    config,
-    local_borg_version,
-    import_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "key import" action for the given repository.
-    '''
-    if import_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, import_arguments.repository
-    ):
-        logger.info('Importing repository key')
-        borgmatic.borg.import_key.import_key(
-            repository['path'],
-            config,
-            local_borg_version,
-            import_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/info.py

@@ -1,50 +0,0 @@
-import logging
-
-import borgmatic.actions.arguments
-import borgmatic.actions.json
-import borgmatic.borg.info
-import borgmatic.borg.repo_list
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_info(
-    repository,
-    config,
-    local_borg_version,
-    info_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "info" action for the given repository and archive.
-
-    If info_arguments.json is True, yield the JSON output from the info for the archive.
-    '''
-    if info_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, info_arguments.repository
-    ):
-        if not info_arguments.json:
-            logger.answer('Displaying archive summary information')
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            repository['path'],
-            info_arguments.archive,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        json_output = borgmatic.borg.info.display_archives_info(
-            repository['path'],
-            config,
-            local_borg_version,
-            borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/json.py

@@ -1,30 +0,0 @@
-import json
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-def parse_json(borg_json_output, label):
-    '''
-    Given a Borg JSON output string, parse it as JSON into a dict. Inject the given borgmatic
-    repository label into it and return the dict.
-
-    Raise JSONDecodeError if the JSON output cannot be parsed.
-    '''
-    lines = borg_json_output.splitlines()
-    start_line_index = 0
-
-    # Scan forward to find the first line starting with "{" and assume that's where the JSON starts.
-    for line_index, line in enumerate(lines):
-        if line.startswith('{'):
-            start_line_index = line_index
-            break
-
-    json_data = json.loads('\n'.join(lines[start_line_index:]))
-
-    if 'repository' not in json_data:
-        return json_data
-
-    json_data['repository']['label'] = label or ''
-
-    return json_data

borgmatic/actions/list.py

@@ -1,53 +0,0 @@
-import logging
-
-import borgmatic.actions.arguments
-import borgmatic.actions.json
-import borgmatic.borg.list
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_list(
-    repository,
-    config,
-    local_borg_version,
-    list_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "list" action for the given repository and archive.
-
-    If list_arguments.json is True, yield the JSON output from listing the archive.
-    '''
-    if list_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, list_arguments.repository
-    ):
-        if not list_arguments.json:
-            if list_arguments.find_paths:  # pragma: no cover
-                logger.answer('Searching archives')
-            elif not list_arguments.archive:  # pragma: no cover
-                logger.answer('Listing archives')
-
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            repository['path'],
-            list_arguments.archive,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        json_output = borgmatic.borg.list.list_archive(
-            repository['path'],
-            config,
-            local_borg_version,
-            borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/mount.py

@@ -1,47 +0,0 @@
-import logging
-
-import borgmatic.borg.mount
-import borgmatic.borg.repo_list
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_mount(
-    repository,
-    config,
-    local_borg_version,
-    mount_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "mount" action for the given repository.
-    '''
-    if mount_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, mount_arguments.repository
-    ):
-        if mount_arguments.archive:
-            logger.info(f'Mounting archive {mount_arguments.archive}')
-        else:  # pragma: nocover
-            logger.info('Mounting repository')
-
-        borgmatic.borg.mount.mount_archive(
-            repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                mount_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            mount_arguments,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/prune.py

@@ -1,39 +0,0 @@
-import logging
-
-import borgmatic.borg.prune
-import borgmatic.config.validate
-import borgmatic.hooks.command
-
-logger = logging.getLogger(__name__)
-
-
-def run_prune(
-    config_filename,
-    repository,
-    config,
-    local_borg_version,
-    prune_arguments,
-    global_arguments,
-    dry_run_label,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "prune" action for the given repository.
-    '''
-    if prune_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, prune_arguments.repository
-    ):
-        return
-
-    logger.info(f'Pruning archives{dry_run_label}')
-    borgmatic.borg.prune.prune_archives(
-        global_arguments.dry_run,
-        repository['path'],
-        config,
-        local_borg_version,
-        prune_arguments,
-        global_arguments,
-        local_path=local_path,
-        remote_path=remote_path,
-    )

borgmatic/actions/recreate.py

@@ -1,53 +0,0 @@
-import logging
-
-import borgmatic.borg.recreate
-import borgmatic.config.validate
-from borgmatic.actions.create import collect_patterns, process_patterns
-
-logger = logging.getLogger(__name__)
-
-
-def run_recreate(
-    repository,
-    config,
-    local_borg_version,
-    recreate_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "recreate" action for the given repository.
-    '''
-    if recreate_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, recreate_arguments.repository
-    ):
-        if recreate_arguments.archive:
-            logger.answer(f'Recreating archive {recreate_arguments.archive}')
-        else:
-            logger.answer('Recreating repository')
-
-        # Collect and process patterns.
-        processed_patterns = process_patterns(
-            collect_patterns(config), borgmatic.config.paths.get_working_directory(config)
-        )
-
-        borgmatic.borg.recreate.recreate_archive(
-            repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                recreate_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            config,
-            local_borg_version,
-            recreate_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            patterns=processed_patterns,
-        )

borgmatic/actions/repo_create.py

@@ -1,41 +0,0 @@
-import logging
-
-import borgmatic.borg.repo_create
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_repo_create(
-    repository,
-    config,
-    local_borg_version,
-    repo_create_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "repo-create" action for the given repository.
-    '''
-    if repo_create_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, repo_create_arguments.repository
-    ):
-        return
-
-    logger.info('Creating repository')
-    borgmatic.borg.repo_create.create_repository(
-        global_arguments.dry_run,
-        repository['path'],
-        config,
-        local_borg_version,
-        global_arguments,
-        repo_create_arguments.encryption_mode,
-        repo_create_arguments.source_repository,
-        repo_create_arguments.copy_crypt_key,
-        repo_create_arguments.append_only,
-        repo_create_arguments.storage_quota,
-        repo_create_arguments.make_parent_dirs,
-        local_path=local_path,
-        remote_path=remote_path,
-    )

borgmatic/actions/repo_delete.py

@@ -1,35 +0,0 @@
-import logging
-
-import borgmatic.borg.repo_delete
-
-logger = logging.getLogger(__name__)
-
-
-def run_repo_delete(
-    repository,
-    config,
-    local_borg_version,
-    repo_delete_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "repo-delete" action for the given repository.
-    '''
-    if repo_delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_delete_arguments.repository
-    ):
-        logger.answer(
-            'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else '')
-        )
-
-        borgmatic.borg.repo_delete.delete_repository(
-            repository,
-            config,
-            local_borg_version,
-            repo_delete_arguments,
-            global_arguments,
-            local_path,
-            remote_path,
-        )

borgmatic/actions/repo_info.py

@@ -1,40 +0,0 @@
-import logging
-
-import borgmatic.actions.json
-import borgmatic.borg.repo_info
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_repo_info(
-    repository,
-    config,
-    local_borg_version,
-    repo_info_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "repo-info" action for the given repository.
-
-    If repo_info_arguments.json is True, yield the JSON output from the info for the repository.
-    '''
-    if repo_info_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_info_arguments.repository
-    ):
-        if not repo_info_arguments.json:
-            logger.answer('Displaying repository summary information')
-
-        json_output = borgmatic.borg.repo_info.display_repository_info(
-            repository['path'],
-            config,
-            local_borg_version,
-            repo_info_arguments=repo_info_arguments,
-            global_arguments=global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/repo_list.py

@@ -1,40 +0,0 @@
-import logging
-
-import borgmatic.actions.json
-import borgmatic.borg.repo_list
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_repo_list(
-    repository,
-    config,
-    local_borg_version,
-    repo_list_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "repo-list" action for the given repository.
-
-    If repo_list_arguments.json is True, yield the JSON output from listing the repository.
-    '''
-    if repo_list_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_list_arguments.repository
-    ):
-        if not repo_list_arguments.json:
-            logger.answer('Listing repository')
-
-        json_output = borgmatic.borg.repo_list.list_repository(
-            repository['path'],
-            config,
-            local_borg_version,
-            repo_list_arguments=repo_list_arguments,
-            global_arguments=global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/restore.py

@@ -1,531 +0,0 @@
-import collections
-import logging
-import os
-import pathlib
-import shutil
-import tempfile
-
-import borgmatic.borg.extract
-import borgmatic.borg.list
-import borgmatic.borg.mount
-import borgmatic.borg.repo_list
-import borgmatic.config.paths
-import borgmatic.config.validate
-import borgmatic.hooks.data_source.dump
-import borgmatic.hooks.dispatch
-
-logger = logging.getLogger(__name__)
-
-
-UNSPECIFIED = object()
-
-
-Dump = collections.namedtuple(
-    'Dump',
-    ('hook_name', 'data_source_name', 'hostname', 'port'),
-    defaults=('localhost', None),
-)
-
-
-def dumps_match(first, second, default_port=None):
-    '''
-    Compare two Dump instances for equality while supporting a field value of UNSPECIFIED, which
-    indicates that the field should match any value. If a default port is given, then consider any
-    dump having that port to match with a dump having a None port.
-    '''
-    for field_name in first._fields:
-        first_value = getattr(first, field_name)
-        second_value = getattr(second, field_name)
-
-        if default_port is not None and field_name == 'port':
-            if first_value == default_port and second_value is None:
-                continue
-            if second_value == default_port and first_value is None:
-                continue
-
-        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:
-            continue
-
-        if first_value != second_value:
-            return False
-
-    return True
-
-
-def render_dump_metadata(dump):
-    '''
-    Given a Dump instance, make a display string describing it for use in log messages.
-    '''
-    name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
-    hostname = dump.hostname or UNSPECIFIED
-    port = None if dump.port is UNSPECIFIED else dump.port
-
-    if port:
-        metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
-    else:
-        metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
-
-    if dump.hook_name not in (None, UNSPECIFIED):
-        return f'{metadata} ({dump.hook_name})'
-
-    return metadata
-
-
-def get_configured_data_source(config, restore_dump):
-    '''
-    Search in the given configuration dict for dumps corresponding to the given dump to restore. If
-    there are multiple matches, error.
-
-    Return the found data source as a data source configuration dict or None if not found.
-    '''
-    try:
-        hooks_to_search = {restore_dump.hook_name: config[restore_dump.hook_name]}
-    except KeyError:
-        return None
-
-    matching_dumps = tuple(
-        hook_data_source
-        for (hook_name, hook_config) in hooks_to_search.items()
-        for hook_data_source in hook_config
-        for default_port in (
-            borgmatic.hooks.dispatch.call_hook(
-                function_name='get_default_port',
-                config=config,
-                hook_name=hook_name,
-            ),
-        )
-        if dumps_match(
-            Dump(
-                hook_name,
-                hook_data_source.get('name'),
-                hook_data_source.get('hostname', 'localhost'),
-                hook_data_source.get('port'),
-            ),
-            restore_dump,
-            default_port,
-        )
-    )
-
-    if not matching_dumps:
-        return None
-
-    if len(matching_dumps) > 1:
-        raise ValueError(
-            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured'
-        )
-
-    return matching_dumps[0]
-
-
-def strip_path_prefix_from_extracted_dump_destination(
-    destination_path, borgmatic_runtime_directory
-):
-    '''
-    Directory-format dump files get extracted into a temporary directory containing a path prefix
-    that depends how the files were stored in the archive. So, given the destination path where the
-    dump was extracted and the borgmatic runtime directory, move the dump files such that the
-    restore doesn't have to deal with that varying path prefix.
-
-    For instance, if the dump was extracted to:
-
-      /run/user/0/borgmatic/tmp1234/borgmatic/postgresql_databases/test/...
-
-    or:
-
-      /run/user/0/borgmatic/tmp1234/root/.borgmatic/postgresql_databases/test/...
-
-    then this function moves it to:
-
-      /run/user/0/borgmatic/postgresql_databases/test/...
-    '''
-    for subdirectory_path, _, _ in os.walk(destination_path):
-        databases_directory = os.path.basename(subdirectory_path)
-
-        if not databases_directory.endswith('_databases'):
-            continue
-
-        shutil.move(
-            subdirectory_path, os.path.join(borgmatic_runtime_directory, databases_directory)
-        )
-        break
-
-
-def restore_single_dump(
-    repository,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path,
-    remote_path,
-    archive_name,
-    hook_name,
-    data_source,
-    connection_params,
-    borgmatic_runtime_directory,
-):
-    '''
-    Given (among other things) an archive name, a data source hook name, the hostname, port,
-    username/password as connection params, and a configured data source configuration dict, restore
-    that data source from the archive.
-    '''
-    dump_metadata = render_dump_metadata(
-        Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'))
-    )
-
-    logger.info(f'Restoring data source {dump_metadata}')
-
-    dump_patterns = borgmatic.hooks.dispatch.call_hooks(
-        'make_data_source_dump_patterns',
-        config,
-        borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-        borgmatic_runtime_directory,
-        data_source['name'],
-    )[hook_name.split('_databases', 1)[0]]
-
-    destination_path = (
-        tempfile.mkdtemp(dir=borgmatic_runtime_directory)
-        if data_source.get('format') == 'directory'
-        else None
-    )
-
-    try:
-        # Kick off a single data source extract. If using a directory format, extract to a temporary
-        # directory. Otherwise extract the single dump file to stdout.
-        extract_process = borgmatic.borg.extract.extract_archive(
-            dry_run=global_arguments.dry_run,
-            repository=repository['path'],
-            archive=archive_name,
-            paths=[
-                borgmatic.hooks.data_source.dump.convert_glob_patterns_to_borg_pattern(
-                    dump_patterns
-                )
-            ],
-            config=config,
-            local_borg_version=local_borg_version,
-            global_arguments=global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            destination_path=destination_path,
-            # A directory format dump isn't a single file, and therefore can't extract
-            # to stdout. In this case, the extract_process return value is None.
-            extract_to_stdout=bool(data_source.get('format') != 'directory'),
-        )
-
-        if destination_path and not global_arguments.dry_run:
-            strip_path_prefix_from_extracted_dump_destination(
-                destination_path, borgmatic_runtime_directory
-            )
-    finally:
-        if destination_path and not global_arguments.dry_run:
-            shutil.rmtree(destination_path, ignore_errors=True)
-
-    # Run a single data source restore, consuming the extract stdout (if any).
-    borgmatic.hooks.dispatch.call_hook(
-        function_name='restore_data_source_dump',
-        config=config,
-        hook_name=hook_name,
-        data_source=data_source,
-        dry_run=global_arguments.dry_run,
-        extract_process=extract_process,
-        connection_params=connection_params,
-        borgmatic_runtime_directory=borgmatic_runtime_directory,
-    )
-
-
-def collect_dumps_from_archive(
-    repository,
-    archive,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path,
-    remote_path,
-    borgmatic_runtime_directory,
-):
-    '''
-    Given a local or remote repository path, a resolved archive name, a configuration dict, the
-    local Borg version, global arguments an argparse.Namespace, local and remote Borg paths, and the
-    borgmatic runtime directory, query the archive for the names of data sources dumps it contains
-    and return them as a set of Dump instances.
-    '''
-    borgmatic_source_directory = str(
-        pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
-    )
-
-    # Probe for the data source dumps in multiple locations, as the default location has moved to
-    # the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
-    # we still want to support reading dumps from previously created archives as well.
-    dump_paths = borgmatic.borg.list.capture_archive_listing(
-        repository,
-        archive,
-        config,
-        local_borg_version,
-        global_arguments,
-        list_paths=[
-            'sh:'
-            + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
-                base_directory, '*_databases/*/*'
-            )
-            for base_directory in (
-                'borgmatic',
-                borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
-                borgmatic_source_directory.lstrip('/'),
-            )
-        ],
-        local_path=local_path,
-        remote_path=remote_path,
-    )
-
-    # Parse the paths of dumps found in the archive to get their respective dump metadata.
-    dumps_from_archive = set()
-
-    for dump_path in dump_paths:
-        if not dump_path:
-            continue
-
-        # Probe to find the base directory that's at the start of the dump path.
-        for base_directory in (
-            'borgmatic',
-            borgmatic_runtime_directory,
-            borgmatic_source_directory,
-        ):
-            try:
-                (hook_name, host_and_port, data_source_name) = dump_path.split(
-                    base_directory + os.path.sep, 1
-                )[1].split(os.path.sep)[0:3]
-            except (ValueError, IndexError):
-                continue
-
-            parts = host_and_port.split(':', 1)
-
-            if len(parts) == 1:
-                parts += (None,)
-
-            (hostname, port) = parts
-
-            try:
-                port = int(port)
-            except (ValueError, TypeError):
-                port = None
-
-            dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port))
-
-            # We've successfully parsed the dump path, so need to probe any further.
-            break
-        else:
-            logger.warning(
-                f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
-            )
-
-    return dumps_from_archive
-
-
-def get_dumps_to_restore(restore_arguments, dumps_from_archive):
-    '''
-    Given restore arguments as an argparse.Namespace instance indicating which dumps to restore and
-    a set of Dump instances representing the dumps found in an archive, return a set of specific
-    Dump instances from the archive to restore. As part of this, replace any Dump having a data
-    source name of "all" with multiple named Dump instances as appropriate.
-
-    Raise ValueError if any of the requested data source names cannot be found in the archive or if
-    there are multiple archive dump matches for a given requested dump.
-    '''
-    requested_dumps = (
-        {
-            Dump(
-                hook_name=(
-                    (
-                        restore_arguments.hook
-                        if restore_arguments.hook.endswith('_databases')
-                        else f'{restore_arguments.hook}_databases'
-                    )
-                    if restore_arguments.hook
-                    else UNSPECIFIED
-                ),
-                data_source_name=name,
-                hostname=restore_arguments.original_hostname or UNSPECIFIED,
-                port=restore_arguments.original_port,
-            )
-            for name in restore_arguments.data_sources or (UNSPECIFIED,)
-        }
-        if restore_arguments.hook
-        or restore_arguments.data_sources
-        or restore_arguments.original_hostname
-        or restore_arguments.original_port
-        else {
-            Dump(
-                hook_name=UNSPECIFIED,
-                data_source_name='all',
-                hostname=UNSPECIFIED,
-                port=UNSPECIFIED,
-            )
-        }
-    )
-    missing_dumps = set()
-    dumps_to_restore = set()
-
-    # If there's a requested "all" dump, add every dump from the archive to the dumps to restore.
-    if any(dump for dump in requested_dumps if dump.data_source_name == 'all'):
-        dumps_to_restore.update(dumps_from_archive)
-
-    # If any archive dump matches a requested dump, add the archive dump to the dumps to restore.
-    for requested_dump in requested_dumps:
-        if requested_dump.data_source_name == 'all':
-            continue
-
-        matching_dumps = tuple(
-            archive_dump
-            for archive_dump in dumps_from_archive
-            if dumps_match(requested_dump, archive_dump)
-        )
-
-        if len(matching_dumps) == 0:
-            missing_dumps.add(requested_dump)
-        elif len(matching_dumps) == 1:
-            dumps_to_restore.add(matching_dumps[0])
-        else:
-            raise ValueError(
-                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.'
-            )
-
-    if missing_dumps:
-        rendered_dumps = ', '.join(
-            f'{render_dump_metadata(dump)}' for dump in sorted(missing_dumps)
-        )
-
-        raise ValueError(
-            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive"
-        )
-
-    return dumps_to_restore
-
-
-def ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored):
-    '''
-    Given a set of requested dumps to restore and a set of dumps actually restored, raise ValueError
-    if any requested dumps to restore weren't restored, indicating that they were missing from the
-    configuration.
-    '''
-    if not dumps_actually_restored:
-        raise ValueError('No data source dumps were found to restore')
-
-    missing_dumps = sorted(
-        dumps_to_restore - dumps_actually_restored, key=lambda dump: dump.data_source_name
-    )
-
-    if missing_dumps:
-        rendered_dumps = ', '.join(f'{render_dump_metadata(dump)}' for dump in missing_dumps)
-
-        raise ValueError(
-            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration"
-        )
-
-
-def run_restore(
-    repository,
-    config,
-    local_borg_version,
-    restore_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "restore" action for the given repository, but only if the repository matches the
-    requested repository in restore arguments.
-
-    Raise ValueError if a configured data source could not be found to restore or there's no
-    matching dump in the archive.
-    '''
-    if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, restore_arguments.repository
-    ):
-        return
-
-    logger.info(f'Restoring data sources from archive {restore_arguments.archive}')
-
-    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )
-
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            repository['path'],
-            restore_arguments.archive,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        dumps_from_archive = collect_dumps_from_archive(
-            repository['path'],
-            archive_name,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-            borgmatic_runtime_directory,
-        )
-        dumps_to_restore = get_dumps_to_restore(restore_arguments, dumps_from_archive)
-
-        dumps_actually_restored = set()
-        connection_params = {
-            'hostname': restore_arguments.hostname,
-            'port': restore_arguments.port,
-            'username': restore_arguments.username,
-            'password': restore_arguments.password,
-            'restore_path': restore_arguments.restore_path,
-        }
-
-        # Restore each dump.
-        for restore_dump in dumps_to_restore:
-            found_data_source = get_configured_data_source(
-                config,
-                restore_dump,
-            )
-
-            # For a dump that wasn't found via an exact match in the configuration, try to fallback
-            # to an "all" data source.
-            if not found_data_source:
-                found_data_source = get_configured_data_source(
-                    config,
-                    Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
-                )
-
-                if not found_data_source:
-                    continue
-
-                found_data_source = dict(found_data_source)
-                found_data_source['name'] = restore_dump.data_source_name
-
-            dumps_actually_restored.add(restore_dump)
-
-            restore_single_dump(
-                repository,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-                archive_name,
-                restore_dump.hook_name,
-                dict(found_data_source, **{'schemas': restore_arguments.schemas}),
-                connection_params,
-                borgmatic_runtime_directory,
-            )
-
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )
-
-    ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored)

borgmatic/actions/transfer.py

@@ -1,30 +0,0 @@
-import logging
-
-import borgmatic.borg.transfer
-
-logger = logging.getLogger(__name__)
-
-
-def run_transfer(
-    repository,
-    config,
-    local_borg_version,
-    transfer_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "transfer" action for the given repository.
-    '''
-    logger.info('Transferring archives to repository')
-    borgmatic.borg.transfer.transfer_archives(
-        global_arguments.dry_run,
-        repository['path'],
-        config,
-        local_borg_version,
-        transfer_arguments,
-        global_arguments,
-        local_path=local_path,
-        remote_path=remote_path,
-    )

borgmatic/borg/borg.py

@@ -1,74 +1,59 @@
 import logging
-import shlex
 
-import borgmatic.commands.arguments
-import borgmatic.config.paths
-import borgmatic.logger
-from borgmatic.borg import environment, flags
-from borgmatic.execute import DO_NOT_CAPTURE, execute_command
+from borgmatic.borg import environment
+from borgmatic.borg.flags import make_flags
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-BORG_SUBCOMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'}
+REPOSITORYLESS_BORG_COMMANDS = {'serve', None}
+BORG_COMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'}
+BORG_SUBCOMMANDS_WITHOUT_REPOSITORY = (('debug', 'info'), ('debug', 'convert-profile'))
 
 
 def run_arbitrary_borg(
-    repository_path,
-    config,
-    local_borg_version,
-    options,
-    archive=None,
-    local_path='borg',
-    remote_path=None,
+    repository, storage_config, options, archive=None, local_path='borg', remote_path=None
 ):
     '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, a
-    sequence of arbitrary command-line Borg options, and an optional archive name, run an arbitrary
-    Borg command, passing in REPOSITORY and ARCHIVE environment variables for optional use in the
-    command.
+    Given a local or remote repository path, a storage config dict, a sequence of arbitrary
+    command-line Borg options, and an optional archive name, run an arbitrary Borg command on the
+    given repository/archive.
     '''
-    borgmatic.logger.add_custom_log_levels()
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = storage_config.get('lock_wait', None)
 
     try:
         options = options[1:] if options[0] == '--' else options
 
         # Borg commands like "key" have a sub-command ("export", etc.) that must follow it.
-        command_options_start_index = 2 if options[0] in BORG_SUBCOMMANDS_WITH_SUBCOMMANDS else 1
+        command_options_start_index = 2 if options[0] in BORG_COMMANDS_WITH_SUBCOMMANDS else 1
         borg_command = tuple(options[:command_options_start_index])
         command_options = tuple(options[command_options_start_index:])
-
-        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES.keys():
-            logger.warning(
-                f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}"
-            )
     except IndexError:
         borg_command = ()
         command_options = ()
 
+    if borg_command in BORG_SUBCOMMANDS_WITHOUT_REPOSITORY:
+        repository_archive = None
+    else:
+        repository_archive = (
+            '::'.join((repository, archive)) if repository and archive else repository
+        )
+
     full_command = (
         (local_path,)
         + borg_command
+        + ((repository_archive,) if borg_command and repository_archive else ())
+        + command_options
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('lock-wait', lock_wait)
-        + command_options
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
     )
 
     return execute_command(
-        tuple(shlex.quote(part) for part in full_command),
-        output_file=DO_NOT_CAPTURE,
-        shell=True,
-        environment=dict(
-            (environment.make_environment(config) or {}),
-            **{
-                'BORG_REPO': repository_path,
-                'ARCHIVE': archive if archive else '',
-            },
-        ),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        full_command,
+        output_log_level=logging.WARNING,
         borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
+        extra_environment=environment.make_environment(storage_config),
     )

borgmatic/borg/break_lock.py

@@ -1,43 +0,0 @@
-import logging
-
-import borgmatic.config.paths
-from borgmatic.borg import environment, flags
-from borgmatic.execute import execute_command
-
-logger = logging.getLogger(__name__)
-
-
-def break_lock(
-    repository_path,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, an
-    argparse.Namespace of global arguments, and optional local and remote Borg paths, break any
-    repository and cache locks leftover from Borg aborting.
-    '''
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
-
-    full_command = (
-        (local_path, 'break-lock')
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + flags.make_repository_flags(repository_path, local_borg_version)
-    )
-
-    execute_command(
-        full_command,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )

borgmatic/borg/change_passphrase.py

@@ -1,67 +0,0 @@
-import logging
-
-import borgmatic.config.paths
-import borgmatic.execute
-import borgmatic.logger
-from borgmatic.borg import environment, flags
-
-logger = logging.getLogger(__name__)
-
-
-def change_passphrase(
-    repository_path,
-    config,
-    local_borg_version,
-    change_passphrase_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, change
-    passphrase arguments, and optional local and remote Borg paths, change the repository passphrase
-    based on an interactive prompt.
-    '''
-    borgmatic.logger.add_custom_log_levels()
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
-
-    full_command = (
-        (local_path, 'key', 'change-passphrase')
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + flags.make_repository_flags(
-            repository_path,
-            local_borg_version,
-        )
-    )
-
-    if global_arguments.dry_run:
-        logger.info('Skipping change password (dry run)')
-        return
-
-    # If the original passphrase is set programmatically, then Borg won't prompt for a new one! So
-    # don't give Borg any passphrase, and it'll ask the user for both old and new ones.
-    config_without_passphrase = {
-        option_name: value
-        for (option_name, value) in config.items()
-        if option_name not in ('encryption_passphrase', 'encryption_passcommand')
-    }
-
-    borgmatic.execute.execute_command(
-        full_command,
-        output_file=borgmatic.execute.DO_NOT_CAPTURE,
-        output_log_level=logging.ANSWER,
-        environment=environment.make_environment(config_without_passphrase),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )
-
-    logger.answer(
-        f"{repository_path}: Don't forget to update your encryption_passphrase option (if needed)"
-    )

borgmatic/borg/check.py

@@ -1,60 +1,159 @@
 import argparse
+import datetime
 import json
 import logging
+import os
+import pathlib
 
-import borgmatic.config.paths
-from borgmatic.borg import environment, feature, flags, repo_info
+from borgmatic.borg import environment, extract, info, state
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
+DEFAULT_CHECKS = (
+    {'name': 'repository', 'frequency': '1 month'},
+    {'name': 'archives', 'frequency': '1 month'},
+)
+DEFAULT_PREFIX = '{hostname}-'
+
+
 logger = logging.getLogger(__name__)
 
 
-def make_archive_filter_flags(local_borg_version, config, checks, check_arguments):
+def parse_checks(consistency_config, only_checks=None):
     '''
-    Given the local Borg version, a configuration dict, a parsed sequence of checks, and check
-    arguments as an argparse.Namespace instance, transform the checks into tuple of command-line
-    flags for filtering archives in a check command.
+    Given a consistency config with a "checks" sequence of dicts and an optional list of override
+    checks, return a tuple of named checks to run.
 
-    If "check_last" is set in the configuration and "archives" is in checks, then include a "--last"
-    flag. And if "prefix" is set in configuration and "archives" is in checks, then include a
-    "--match-archives" flag.
+    For example, given a retention config of:
+
+        {'checks': ({'name': 'repository'}, {'name': 'archives'})}
+
+    This will be returned as:
+
+        ('repository', 'archives')
+
+    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If a checks value
+    has a name of "disabled", return an empty tuple, meaning that no checks should be run.
+
+    If the "data" check is present, then make sure the "archives" check is included as well.
     '''
-    check_last = config.get('check_last', None)
-    prefix = config.get('prefix')
-
-    if 'archives' in checks or 'data' in checks:
-        return (('--last', str(check_last)) if check_last else ()) + (
-            (
-                ('--match-archives', f'sh:{prefix}*')
-                if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
-                else ('--glob-archives', f'{prefix}*')
+    checks = only_checks or tuple(
+        check_config['name']
+        for check_config in (consistency_config.get('checks', None) or DEFAULT_CHECKS)
+    )
+    checks = tuple(check.lower() for check in checks)
+    if 'disabled' in checks:
+        if len(checks) > 1:
+            logger.warning(
+                'Multiple checks are configured, but one of them is "disabled"; not running any checks'
             )
-            if prefix
-            else (
-                flags.make_match_archives_flags(
-                    check_arguments.match_archives or config.get('match_archives'),
-                    config.get('archive_name_format'),
-                    local_borg_version,
-                )
-            )
-        )
+        return ()
 
-    if check_last:
-        logger.warning(
-            'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
-        )
-    if prefix:
-        logger.warning(
-            'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
-        )
+    if 'data' in checks and 'archives' not in checks:
+        return checks + ('archives',)
 
-    return ()
+    return checks
 
 
-def make_check_name_flags(checks, archive_filter_flags):
+def parse_frequency(frequency):
     '''
-    Given parsed checks set and a sequence of flags to filter archives, transform the checks into
-    tuple of command-line check flags.
+    Given a frequency string with a number and a unit of time, return a corresponding
+    datetime.timedelta instance or None if the frequency is None or "always".
+
+    For instance, given "3 weeks", return datetime.timedelta(weeks=3)
+
+    Raise ValueError if the given frequency cannot be parsed.
+    '''
+    if not frequency:
+        return None
+
+    frequency = frequency.strip().lower()
+
+    if frequency == 'always':
+        return None
+
+    try:
+        number, time_unit = frequency.split(' ')
+        number = int(number)
+    except ValueError:
+        raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
+
+    if not time_unit.endswith('s'):
+        time_unit += 's'
+
+    if time_unit == 'months':
+        number *= 30
+        time_unit = 'days'
+    elif time_unit == 'years':
+        number *= 365
+        time_unit = 'days'
+
+    try:
+        return datetime.timedelta(**{time_unit: number})
+    except TypeError:
+        raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
+
+
+def filter_checks_on_frequency(
+    location_config, consistency_config, borg_repository_id, checks, force
+):
+    '''
+    Given a location config, a consistency config with a "checks" sequence of dicts, a Borg
+    repository ID, a sequence of checks, and whether to force checks to run, filter down those
+    checks based on the configured "frequency" for each check as compared to its check time file.
+
+    In other words, a check whose check time file's timestamp is too new (based on the configured
+    frequency) will get cut from the returned sequence of checks. Example:
+
+    consistency_config = {
+        'checks': [
+            {
+                'name': 'archives',
+                'frequency': '2 weeks',
+            },
+        ]
+    }
+
+    When this function is called with that consistency_config and "archives" in checks, "archives"
+    will get filtered out of the returned result if its check time file is newer than 2 weeks old,
+    indicating that it's not yet time to run that check again.
+
+    Raise ValueError if a frequency cannot be parsed.
+    '''
+    filtered_checks = list(checks)
+
+    if force:
+        return tuple(filtered_checks)
+
+    for check_config in consistency_config.get('checks', DEFAULT_CHECKS):
+        check = check_config['name']
+        if checks and check not in checks:
+            continue
+
+        frequency_delta = parse_frequency(check_config.get('frequency'))
+        if not frequency_delta:
+            continue
+
+        check_time = read_check_time(
+            make_check_time_path(location_config, borg_repository_id, check)
+        )
+        if not check_time:
+            continue
+
+        # If we've not yet reached the time when the frequency dictates we're ready for another
+        # check, skip this check.
+        if datetime.datetime.now() < check_time + frequency_delta:
+            remaining = check_time + frequency_delta - datetime.datetime.now()
+            logger.info(
+                f"Skipping {check} check due to configured frequency; {remaining} until next check"
+            )
+            filtered_checks.remove(check)
+
+    return tuple(filtered_checks)
+
+
+def make_check_flags(checks, check_last=None, prefix=None):
+    '''
+    Given a parsed sequence of checks, transform it into tuple of command-line flags.
 
     For example, given parsed checks of:
 
@@ -64,126 +163,163 @@ def make_check_name_flags(checks, archive_filter_flags):
 
         ('--repository-only',)
 
-    However, if both "repository" and "archives" are in checks, then omit the "only" flags from the
-    returned flags because Borg does both checks by default. Note that a "data" check only works
-    along with an "archives" check.
-    '''
-    data_flags = ('--verify-data',) if 'data' in checks else ()
-    common_flags = (archive_filter_flags if 'archives' in checks else ()) + data_flags
+    However, if both "repository" and "archives" are in checks, then omit them from the returned
+    flags because Borg does both checks by default.
 
-    if {'repository', 'archives'}.issubset(checks):
+    Additionally, if a check_last value is given and "archives" is in checks, then include a
+    "--last" flag. And if a prefix value is given and "archives" is in checks, then include a
+    "--prefix" flag.
+    '''
+    if 'archives' in checks:
+        last_flags = ('--last', str(check_last)) if check_last else ()
+        prefix_flags = ('--prefix', prefix) if prefix else ()
+    else:
+        last_flags = ()
+        prefix_flags = ()
+        if check_last:
+            logger.info('Ignoring check_last option, as "archives" is not in consistency checks')
+        if prefix:
+            logger.info(
+                'Ignoring consistency prefix option, as "archives" is not in consistency checks'
+            )
+
+    common_flags = last_flags + prefix_flags + (('--verify-data',) if 'data' in checks else ())
+
+    if {'repository', 'archives'}.issubset(set(checks)):
         return common_flags
 
     return (
-        tuple(f'--{check}-only' for check in checks if check in ('repository', 'archives'))
+        tuple('--{}-only'.format(check) for check in checks if check in ('repository', 'archives'))
         + common_flags
     )
 
 
-def get_repository_id(
-    repository_path, config, local_borg_version, global_arguments, local_path, remote_path
+def make_check_time_path(location_config, borg_repository_id, check_type):
+    '''
+    Given a location configuration dict, a Borg repository ID, and the name of a check type
+    ("repository", "archives", etc.), return a path for recording that check's time (the time of
+    that check last occurring).
+    '''
+    return os.path.join(
+        os.path.expanduser(
+            location_config.get(
+                'borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+            )
+        ),
+        'checks',
+        borg_repository_id,
+        check_type,
+    )
+
+
+def write_check_time(path):  # pragma: no cover
+    '''
+    Record a check time of now as the modification time of the given path.
+    '''
+    logger.debug(f'Writing check time at {path}')
+
+    os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
+    pathlib.Path(path, mode=0o600).touch()
+
+
+def read_check_time(path):
+    '''
+    Return the check time based on the modification time of the given path. Return None if the path
+    doesn't exist.
+    '''
+    logger.debug(f'Reading check time from {path}')
+
+    try:
+        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
+    except FileNotFoundError:
+        return None
+
+
+def check_archives(
+    repository,
+    location_config,
+    storage_config,
+    consistency_config,
+    local_path='borg',
+    remote_path=None,
+    progress=None,
+    repair=None,
+    only_checks=None,
+    force=None,
 ):
     '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, global
-    arguments, and local/remote commands to run, return the corresponding Borg repository ID.
+    Given a local or remote repository path, a storage config dict, a consistency config dict,
+    local/remote commands to run, whether to include progress information, whether to attempt a
+    repair, and an optional list of checks to use instead of configured checks, check the contained
+    Borg archives for consistency.
 
-    Raise ValueError if the Borg repository ID cannot be determined.
+    If there are no consistency checks to run, skip running them.
+
+    Raises ValueError if the Borg repository ID cannot be determined.
     '''
     try:
-        return json.loads(
-            repo_info.display_repository_info(
-                repository_path,
-                config,
-                local_borg_version,
-                argparse.Namespace(json=True),
-                global_arguments,
+        borg_repository_id = json.loads(
+            info.display_archives_info(
+                repository,
+                storage_config,
+                argparse.Namespace(json=True, archive=None),
                 local_path,
                 remote_path,
             )
         )['repository']['id']
     except (json.JSONDecodeError, KeyError):
-        raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
+        raise ValueError(f'Cannot determine Borg repository ID for {repository}')
 
+    checks = filter_checks_on_frequency(
+        location_config,
+        consistency_config,
+        borg_repository_id,
+        parse_checks(consistency_config, only_checks),
+        force,
+    )
+    check_last = consistency_config.get('check_last', None)
+    lock_wait = None
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
 
-def check_archives(
-    repository_path,
-    config,
-    local_borg_version,
-    check_arguments,
-    global_arguments,
-    checks,
-    archive_filter_flags,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, check
-    arguments as an argparse.Namespace instance, global arguments, a set of named Borg checks to run
-    (some combination "repository", "archives", and/or "data"), archive filter flags, and
-    local/remote commands to run, check the contained Borg archives for consistency.
-    '''
-    lock_wait = config.get('lock_wait')
-    extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
+    if set(checks).intersection({'repository', 'archives', 'data'}):
+        lock_wait = storage_config.get('lock_wait', None)
 
-    verbosity_flags = ()
-    if logger.isEnabledFor(logging.INFO):
-        verbosity_flags = ('--info',)
-    if logger.isEnabledFor(logging.DEBUG):
-        verbosity_flags = ('--debug', '--show-rc')
+        verbosity_flags = ()
+        if logger.isEnabledFor(logging.INFO):
+            verbosity_flags = ('--info',)
+        if logger.isEnabledFor(logging.DEBUG):
+            verbosity_flags = ('--debug', '--show-rc')
 
-    try:
-        repository_check_config = next(
-            check for check in config.get('checks', ()) if check.get('name') == 'repository'
-        )
-    except StopIteration:
-        repository_check_config = {}
+        prefix = consistency_config.get('prefix', DEFAULT_PREFIX)
 
-    max_duration = check_arguments.max_duration or repository_check_config.get('max_duration')
-
-    umask = config.get('umask')
-    borg_exit_codes = config.get('borg_exit_codes')
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    if 'data' in checks:
-        checks.add('archives')
-
-    grouped_checks = (checks,)
-
-    # If max_duration is set, then archives and repository checks need to be run separately, as Borg
-    # doesn't support --max-duration along with an archives checks.
-    if max_duration and 'archives' in checks and 'repository' in checks:
-        checks.remove('repository')
-        grouped_checks = (checks, {'repository'})
-
-    for checks_subset in grouped_checks:
         full_command = (
             (local_path, 'check')
-            + (('--repair',) if check_arguments.repair else ())
-            + (
-                ('--max-duration', str(max_duration))
-                if max_duration and 'repository' in checks_subset
-                else ()
-            )
-            + make_check_name_flags(checks_subset, archive_filter_flags)
+            + (('--repair',) if repair else ())
+            + make_check_flags(checks, check_last, prefix)
             + (('--remote-path', remote_path) if remote_path else ())
-            + (('--umask', str(umask)) if umask else ())
-            + (('--log-json',) if global_arguments.log_json else ())
             + (('--lock-wait', str(lock_wait)) if lock_wait else ())
             + verbosity_flags
-            + (('--progress',) if check_arguments.progress else ())
+            + (('--progress',) if progress else ())
             + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-            + flags.make_repository_flags(repository_path, local_borg_version)
+            + (repository,)
         )
 
-        execute_command(
-            full_command,
-            # The Borg repair option triggers an interactive prompt, which won't work when output is
-            # captured. And progress messes with the terminal directly.
-            output_file=(
-                DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None
-            ),
-            environment=environment.make_environment(config),
-            working_directory=working_directory,
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+        borg_environment = environment.make_environment(storage_config)
+
+        # The Borg repair option triggers an interactive prompt, which won't work when output is
+        # captured. And progress messes with the terminal directly.
+        if repair or progress:
+            execute_command(
+                full_command, output_file=DO_NOT_CAPTURE, extra_environment=borg_environment
+            )
+        else:
+            execute_command(full_command, extra_environment=borg_environment)
+
+        for check in checks:
+            write_check_time(make_check_time_path(location_config, borg_repository_id, check))
+
+    if 'extract' in checks:
+        extract.extract_last_archive_dry_run(
+            storage_config, repository, lock_wait, local_path, remote_path
         )
+        write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract'))

borgmatic/borg/compact.py

@@ -1,7 +1,6 @@
 import logging
 
-import borgmatic.config.paths
-from borgmatic.borg import environment, flags
+from borgmatic.borg import environment
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
@@ -9,10 +8,8 @@ logger = logging.getLogger(__name__)
 
 def compact_segments(
     dry_run,
-    repository_path,
-    config,
-    local_borg_version,
-    global_arguments,
+    repository,
+    storage_config,
     local_path='borg',
     remote_path=None,
     progress=False,
@@ -20,18 +17,17 @@ def compact_segments(
     threshold=None,
 ):
     '''
-    Given dry-run flag, a local or remote repository path, a configuration dict, and the local Borg
-    version, compact the segments in a repository.
+    Given dry-run flag, a local or remote repository path, and a storage config dict, compact Borg
+    segments in a repository.
     '''
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
-    extra_borg_options = config.get('extra_borg_options', {}).get('compact', '')
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('compact', '')
 
     full_command = (
         (local_path, 'compact')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--progress',) if progress else ())
         + (('--cleanup-commits',) if cleanup_commits else ())
@@ -39,18 +35,13 @@ def compact_segments(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_flags(repository_path, local_borg_version)
+        + (repository,)
     )
 
-    if dry_run:
-        logging.info('Skipping compact (dry run)')
-        return
-
-    execute_command(
-        full_command,
-        output_log_level=logging.INFO,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )
+    if not dry_run:
+        execute_command(
+            full_command,
+            output_log_level=logging.INFO,
+            borg_local_path=local_path,
+            extra_environment=environment.make_environment(storage_config),
+        )

borgmatic/borg/create.py

@@ -1,404 +1,315 @@
+import glob
 import itertools
 import logging
 import os
 import pathlib
-import stat
 import tempfile
-import textwrap
 
-import borgmatic.borg.pattern
-import borgmatic.config.paths
-import borgmatic.logger
-from borgmatic.borg import environment, feature, flags
-from borgmatic.execute import (
-    DO_NOT_CAPTURE,
-    execute_command,
-    execute_command_and_capture_output,
-    execute_command_with_processes,
-)
+from borgmatic.borg import environment, feature, state
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
 
 logger = logging.getLogger(__name__)
 
 
-def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None):
+def expand_directory(directory):
     '''
-    Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
-    temporary file in the given borgmatic runtime directory and return the file object so it can
-    continue to exist on disk as long as the caller needs it.
+    Given a directory path, expand any tilde (representing a user's home directory) and any globs
+    therein. Return a list of one or more resulting paths.
+    '''
+    expanded_directory = os.path.expanduser(directory)
 
-    If an optional open pattern file is given, append to it instead of making a new temporary file.
-    Return None if no patterns are provided.
+    return glob.glob(expanded_directory) or [expanded_directory]
+
+
+def expand_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes and globs in each one. Return all the
+    resulting directories as a single flattened tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(expand_directory(directory) for directory in directories)
+    )
+
+
+def expand_home_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
+    Return the results as a tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(os.path.expanduser(directory) for directory in directories)
+
+
+def map_directories_to_devices(directories):
+    '''
+    Given a sequence of directories, return a map from directory to an identifier for the device on
+    which that directory resides or None if the path doesn't exist.
+
+    This is handy for determining whether two different directories are on the same filesystem (have
+    the same device identifier).
+    '''
+    return {
+        directory: os.stat(directory).st_dev if os.path.exists(directory) else None
+        for directory in directories
+    }
+
+
+def deduplicate_directories(directory_devices):
+    '''
+    Given a map from directory to the identifier for the device on which that directory resides,
+    return the directories as a sorted tuple with all duplicate child directories removed. For
+    instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
+
+    The one exception to this rule is if two paths are on different filesystems (devices). In that
+    case, they won't get de-duplicated in case they both need to be passed to Borg (e.g. the
+    location.one_file_system option is true).
+
+    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
+    child directories, because it will naturally spider the contents of the parent directory. And
+    there are cases where Borg coming across the same file twice will result in duplicate reads and
+    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
+    Borg.
+    '''
+    deduplicated = set()
+    directories = sorted(directory_devices.keys())
+
+    for directory in directories:
+        deduplicated.add(directory)
+        parents = pathlib.PurePath(directory).parents
+
+        # If another directory in the given list is a parent of current directory (even n levels
+        # up) and both are on the same filesystem, then the current directory is a duplicate.
+        for other_directory in directories:
+            for parent in parents:
+                if (
+                    pathlib.PurePath(other_directory) == parent
+                    and directory_devices[directory] is not None
+                    and directory_devices[other_directory] == directory_devices[directory]
+                ):
+                    if directory in deduplicated:
+                        deduplicated.remove(directory)
+                    break
+
+    return tuple(sorted(deduplicated))
+
+
+def write_pattern_file(patterns=None):
+    '''
+    Given a sequence of patterns, write them to a named temporary file and return it. Return None
+    if no patterns are provided.
     '''
     if not patterns:
         return None
 
-    if patterns_file is None:
-        patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
-        operation_name = 'Writing'
-    else:
-        patterns_file.write('\n')
-        operation_name = 'Appending'
+    pattern_file = tempfile.NamedTemporaryFile('w')
+    pattern_file.write('\n'.join(patterns))
+    pattern_file.flush()
 
-    patterns_output = '\n'.join(
-        f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}'
-        for pattern in patterns
+    return pattern_file
+
+
+def ensure_files_readable(*filename_lists):
+    '''
+    Given a sequence of filename sequences, ensure that each filename is openable. This prevents
+    unreadable files from being passed to Borg, which in certain situations only warns instead of
+    erroring.
+    '''
+    for file_object in itertools.chain.from_iterable(
+        filename_list for filename_list in filename_lists if filename_list
+    ):
+        open(file_object).close()
+
+
+def make_pattern_flags(location_config, pattern_filename=None):
+    '''
+    Given a location config dict with a potential patterns_from option, and a filename containing
+    any additional patterns, return the corresponding Borg flags for those files as a tuple.
+    '''
+    pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
+        (pattern_filename,) if pattern_filename else ()
     )
-    logger.debug(f'{operation_name} patterns to {patterns_file.name}:\n{patterns_output}')
 
-    patterns_file.write(patterns_output)
-    patterns_file.flush()
-
-    return patterns_file
+    return tuple(
+        itertools.chain.from_iterable(
+            ('--patterns-from', pattern_filename) for pattern_filename in pattern_filenames
+        )
+    )
 
 
-def make_exclude_flags(config):
+def make_exclude_flags(location_config, exclude_filename=None):
     '''
-    Given a configuration dict with various exclude options, return the corresponding Borg flags as
-    a tuple.
+    Given a location config dict with various exclude options, and a filename containing any exclude
+    patterns, return the corresponding Borg flags as a tuple.
     '''
-    caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else ()
+    exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
+        (exclude_filename,) if exclude_filename else ()
+    )
+    exclude_from_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-from', exclude_filename) for exclude_filename in exclude_filenames
+        )
+    )
+    caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
     if_present_flags = tuple(
         itertools.chain.from_iterable(
             ('--exclude-if-present', if_present)
-            for if_present in config.get('exclude_if_present', ())
+            for if_present in location_config.get('exclude_if_present', ())
         )
     )
-    keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
-    exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
-
-    return caches_flag + if_present_flags + keep_exclude_tags_flags + exclude_nodump_flags
-
-
-def make_list_filter_flags(local_borg_version, dry_run):
-    '''
-    Given the local Borg version and whether this is a dry run, return the corresponding flags for
-    passing to "--list --filter". The general idea is that excludes are shown for a dry run or when
-    the verbosity is debug.
-    '''
-    base_flags = 'AME'
-    show_excludes = logger.isEnabledFor(logging.DEBUG)
-
-    if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version):
-        if show_excludes or dry_run:
-            return f'{base_flags}+-'
-        else:
-            return base_flags
-
-    if show_excludes:
-        return f'{base_flags}x-'
-    else:
-        return f'{base_flags}-'
-
-
-def special_file(path, working_directory=None):
-    '''
-    Return whether the given path is a special file (character device, block device, or named pipe
-    / FIFO). If a working directory is given, take it into account when making the full path to
-    check.
-    '''
-    try:
-        mode = os.stat(os.path.join(working_directory or '', path)).st_mode
-    except (FileNotFoundError, OSError):
-        return False
-
-    return stat.S_ISCHR(mode) or stat.S_ISBLK(mode) or stat.S_ISFIFO(mode)
-
-
-def any_parent_directories(path, candidate_parents):
-    '''
-    Return whether any of the given candidate parent directories are an actual parent of the given
-    path. This includes grandparents, etc.
-    '''
-    for parent in candidate_parents:
-        if pathlib.PurePosixPath(parent) in pathlib.PurePath(path).parents:
-            return True
-
-    return False
-
-
-def collect_special_file_paths(
-    dry_run,
-    create_command,
-    config,
-    local_path,
-    working_directory,
-    borgmatic_runtime_directory,
-):
-    '''
-    Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
-    a working directory, and the borgmatic runtime directory, collect the paths for any special
-    files (character devices, block devices, and named pipes / FIFOs) that Borg would encounter
-    during a create. These are all paths that could cause Borg to hang if its --read-special flag is
-    used.
-
-    Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
-    its own special files there for database dumps and we don't want those omitted.
-
-    Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg
-    plans to backup, that means the user must have excluded the runtime directory (e.g. via
-    "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to
-    consume any database dumps and therefore borgmatic will hang when it tries to do so.
-    '''
-    # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
-    # files including any named pipe we've created. And omit "--filter" because that can break the
-    # paths output parsing below such that path lines no longer start with th expected "- ".
-    paths_output = execute_command_and_capture_output(
-        flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter')
-        + ('--dry-run', '--list'),
-        capture_stderr=True,
-        working_directory=working_directory,
-        environment=environment.make_environment(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
+    keep_exclude_tags_flags = (
+        ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else ()
     )
+    exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else ()
 
-    # These are all the individual files that Borg is planning to backup as determined by the Borg
-    # create dry run above.
-    paths = tuple(
-        path_line.split(' ', 1)[1]
-        for path_line in paths_output.split('\n')
-        if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
-    )
-
-    # These are the subset of those files that contain the borgmatic runtime directory.
-    paths_containing_runtime_directory = {}
-
-    if os.path.exists(borgmatic_runtime_directory):
-        paths_containing_runtime_directory = {
-            path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
-        }
-
-        # If no paths to backup contain the runtime directory, it must've been excluded.
-        if not paths_containing_runtime_directory and not dry_run:
-            raise ValueError(
-                f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
-            )
-
-    return tuple(
-        path
-        for path in paths
-        if special_file(path, working_directory)
-        if path not in paths_containing_runtime_directory
+    return (
+        exclude_from_flags
+        + caches_flag
+        + if_present_flags
+        + keep_exclude_tags_flags
+        + exclude_nodump_flags
     )
 
 
-def check_all_root_patterns_exist(patterns):
+DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+
+
+def borgmatic_source_directories(borgmatic_source_directory):
     '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
-    paths exist. If any don't, raise an exception.
+    Return a list of borgmatic-specific source directories used for state like database backups.
     '''
-    missing_paths = [
-        pattern.path
-        for pattern in patterns
-        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-        if not os.path.exists(pattern.path)
-    ]
+    if not borgmatic_source_directory:
+        borgmatic_source_directory = state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
 
-    if missing_paths:
-        raise ValueError(
-            f"Source directories / root pattern paths do not exist: {', '.join(missing_paths)}"
-        )
-
-
-MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
-
-
-def make_base_create_command(
-    dry_run,
-    repository_path,
-    config,
-    patterns,
-    local_borg_version,
-    global_arguments,
-    borgmatic_runtime_directory,
-    local_path='borg',
-    remote_path=None,
-    progress=False,
-    json=False,
-    list_files=False,
-    stream_processes=None,
-):
-    '''
-    Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version,
-    global arguments as an argparse.Namespace instance, and a sequence of borgmatic source
-    directories, return a tuple of (base Borg create command flags, Borg create command positional
-    arguments, open pattern file handle).
-    '''
-    if config.get('source_directories_must_exist', False):
-        check_all_root_patterns_exist(patterns)
-
-    patterns_file = write_patterns_file(patterns, borgmatic_runtime_directory)
-    checkpoint_interval = config.get('checkpoint_interval', None)
-    checkpoint_volume = config.get('checkpoint_volume', None)
-    chunker_params = config.get('chunker_params', None)
-    compression = config.get('compression', None)
-    upload_rate_limit = config.get('upload_rate_limit', None)
-    upload_buffer_size = config.get('upload_buffer_size', None)
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
-    list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
-    files_cache = config.get('files_cache')
-    archive_name_format = config.get(
-        'archive_name_format', flags.get_default_archive_name_format(local_borg_version)
+    return (
+        [borgmatic_source_directory]
+        if os.path.exists(os.path.expanduser(borgmatic_source_directory))
+        else []
     )
-    extra_borg_options = config.get('extra_borg_options', {}).get('create', '')
-
-    if feature.available(feature.Feature.ATIME, local_borg_version):
-        atime_flags = ('--atime',) if config.get('atime') is True else ()
-    else:
-        atime_flags = ('--noatime',) if config.get('atime') is False else ()
-
-    if feature.available(feature.Feature.NOFLAGS, local_borg_version):
-        noflags_flags = ('--noflags',) if config.get('flags') is False else ()
-    else:
-        noflags_flags = ('--nobsdflags',) if config.get('flags') is False else ()
-
-    if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version):
-        numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
-    else:
-        numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else ()
-
-    if feature.available(feature.Feature.UPLOAD_RATELIMIT, local_borg_version):
-        upload_ratelimit_flags = (
-            ('--upload-ratelimit', str(upload_rate_limit)) if upload_rate_limit else ()
-        )
-    else:
-        upload_ratelimit_flags = (
-            ('--remote-ratelimit', str(upload_rate_limit)) if upload_rate_limit else ()
-        )
-
-    create_flags = (
-        tuple(local_path.split(' '))
-        + ('create',)
-        + (('--patterns-from', patterns_file.name) if patterns_file else ())
-        + make_exclude_flags(config)
-        + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
-        + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ())
-        + (('--chunker-params', chunker_params) if chunker_params else ())
-        + (('--compression', compression) if compression else ())
-        + upload_ratelimit_flags
-        + (('--upload-buffer', str(upload_buffer_size)) if upload_buffer_size else ())
-        + (('--one-file-system',) if config.get('one_file_system') else ())
-        + numeric_ids_flags
-        + atime_flags
-        + (('--noctime',) if config.get('ctime') is False else ())
-        + (('--nobirthtime',) if config.get('birthtime') is False else ())
-        + (('--read-special',) if config.get('read_special') or stream_processes else ())
-        + noflags_flags
-        + (('--files-cache', files_cache) if files_cache else ())
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (
-            ('--list', '--filter', list_filter_flags)
-            if list_files and not json and not progress
-            else ()
-        )
-        + (('--dry-run',) if dry_run else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-    )
-
-    create_positional_arguments = flags.make_repository_archive_flags(
-        repository_path, archive_name_format, local_borg_version
-    )
-
-    # If database hooks are enabled (as indicated by streaming processes), exclude files that might
-    # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
-    if stream_processes and not config.get('read_special'):
-        logger.warning(
-            'Ignoring configured "read_special" value of false, as true is needed for database hooks.'
-        )
-        working_directory = borgmatic.config.paths.get_working_directory(config)
-
-        logger.debug('Collecting special file paths')
-        special_file_paths = collect_special_file_paths(
-            dry_run,
-            create_flags + create_positional_arguments,
-            config,
-            local_path,
-            working_directory,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
-        )
-
-        if special_file_paths:
-            truncated_special_file_paths = textwrap.shorten(
-                ', '.join(special_file_paths),
-                width=MAX_SPECIAL_FILE_PATHS_LENGTH,
-                placeholder=' ...',
-            )
-            logger.warning(
-                f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}'
-            )
-            patterns_file = write_patterns_file(
-                tuple(
-                    borgmatic.borg.pattern.Pattern(
-                        special_file_path,
-                        borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
-                        borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                        source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
-                    )
-                    for special_file_path in special_file_paths
-                ),
-                borgmatic_runtime_directory,
-                patterns_file=patterns_file,
-            )
-
-            if '--patterns-from' not in create_flags:
-                create_flags += ('--patterns-from', patterns_file.name)
-
-    return (create_flags, create_positional_arguments, patterns_file)
 
 
 def create_archive(
     dry_run,
-    repository_path,
-    config,
-    patterns,
+    repository,
+    location_config,
+    storage_config,
     local_borg_version,
-    global_arguments,
-    borgmatic_runtime_directory,
     local_path='borg',
     remote_path=None,
     progress=False,
     stats=False,
     json=False,
-    list_files=False,
+    files=False,
     stream_processes=None,
 ):
     '''
-    Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of loaded configuration paths, the local Borg version, and global arguments as an
-    argparse.Namespace instance, create a Borg archive and return Borg's JSON output (if any).
+    Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
+    storage config dict, create a Borg archive and return Borg's JSON output (if any).
 
     If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
     create command while also triggering the given processes to produce output.
     '''
-    borgmatic.logger.add_custom_log_levels()
+    sources = deduplicate_directories(
+        map_directories_to_devices(
+            expand_directories(
+                location_config['source_directories']
+                + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+            )
+        )
+    )
 
-    working_directory = borgmatic.config.paths.get_working_directory(config)
+    try:
+        working_directory = os.path.expanduser(location_config.get('working_directory'))
+    except TypeError:
+        working_directory = None
+    pattern_file = write_pattern_file(location_config.get('patterns'))
+    exclude_file = write_pattern_file(
+        expand_home_directories(location_config.get('exclude_patterns'))
+    )
+    checkpoint_interval = storage_config.get('checkpoint_interval', None)
+    chunker_params = storage_config.get('chunker_params', None)
+    compression = storage_config.get('compression', None)
+    remote_rate_limit = storage_config.get('remote_rate_limit', None)
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+    files_cache = location_config.get('files_cache')
+    archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
 
-    (create_flags, create_positional_arguments, patterns_file) = make_base_create_command(
-        dry_run,
-        repository_path,
-        config,
-        patterns,
-        local_borg_version,
-        global_arguments,
-        borgmatic_runtime_directory,
-        local_path,
-        remote_path,
-        progress,
-        json,
-        list_files,
-        stream_processes,
+    if feature.available(feature.Feature.ATIME, local_borg_version):
+        atime_flags = ('--atime',) if location_config.get('atime') is True else ()
+    else:
+        atime_flags = ('--noatime',) if location_config.get('atime') is False else ()
+
+    if feature.available(feature.Feature.NOFLAGS, local_borg_version):
+        noflags_flags = ('--noflags',) if location_config.get('bsd_flags') is False else ()
+    else:
+        noflags_flags = ('--nobsdflags',) if location_config.get('bsd_flags') is False else ()
+
+    if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version):
+        numeric_ids_flags = ('--numeric-ids',) if location_config.get('numeric_owner') else ()
+    else:
+        numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_owner') else ()
+
+    if feature.available(feature.Feature.UPLOAD_RATELIMIT, local_borg_version):
+        upload_ratelimit_flags = (
+            ('--upload-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()
+        )
+    else:
+        upload_ratelimit_flags = (
+            ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()
+        )
+
+    ensure_files_readable(location_config.get('patterns_from'), location_config.get('exclude_from'))
+
+    full_command = (
+        tuple(local_path.split(' '))
+        + ('create',)
+        + make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
+        + make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
+        + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
+        + (('--chunker-params', chunker_params) if chunker_params else ())
+        + (('--compression', compression) if compression else ())
+        + upload_ratelimit_flags
+        + (
+            ('--one-file-system',)
+            if location_config.get('one_file_system') or stream_processes
+            else ()
+        )
+        + numeric_ids_flags
+        + atime_flags
+        + (('--noctime',) if location_config.get('ctime') is False else ())
+        + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
+        + (('--read-special',) if (location_config.get('read_special') or stream_processes) else ())
+        + noflags_flags
+        + (('--files-cache', files_cache) if files_cache else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--list', '--filter', 'AME-') if files and not json and not progress else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
+        + (('--stats',) if stats and not json and not dry_run else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
+        + (('--json',) if json else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (
+            '{repository}::{archive_name_format}'.format(
+                repository=repository, archive_name_format=archive_name_format
+            ),
+        )
+        + sources
     )
 
     if json:
         output_log_level = None
-    elif list_files or (stats and not dry_run):
-        output_log_level = logging.ANSWER
+    elif (stats or files) and logger.getEffectiveLevel() == logging.WARNING:
+        output_log_level = logging.WARNING
     else:
         output_log_level = logging.INFO
 
@@ -406,41 +317,24 @@ def create_archive(
     # the terminal directly.
     output_file = DO_NOT_CAPTURE if progress else None
 
-    create_flags += (
-        (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
-        + (('--stats',) if stats and not json and not dry_run else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
-        + (('--progress',) if progress else ())
-        + (('--json',) if json else ())
-    )
-    borg_exit_codes = config.get('borg_exit_codes')
+    borg_environment = environment.make_environment(storage_config)
 
     if stream_processes:
         return execute_command_with_processes(
-            create_flags + create_positional_arguments,
+            full_command,
             stream_processes,
             output_log_level,
             output_file,
-            working_directory=working_directory,
-            environment=environment.make_environment(config),
             borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
-        )
-    elif output_log_level is None:
-        return execute_command_and_capture_output(
-            create_flags + create_positional_arguments,
             working_directory=working_directory,
-            environment=environment.make_environment(config),
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
-        )
-    else:
-        execute_command(
-            create_flags + create_positional_arguments,
-            output_log_level,
-            output_file,
-            working_directory=working_directory,
-            environment=environment.make_environment(config),
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+            extra_environment=borg_environment,
         )
+
+    return execute_command(
+        full_command,
+        output_log_level,
+        output_file,
+        borg_local_path=local_path,
+        working_directory=working_directory,
+        extra_environment=borg_environment,
+    )

borgmatic/borg/delete.py

@@ -1,135 +0,0 @@
-import argparse
-import logging
-
-import borgmatic.borg.environment
-import borgmatic.borg.feature
-import borgmatic.borg.flags
-import borgmatic.borg.repo_delete
-import borgmatic.config.paths
-import borgmatic.execute
-
-logger = logging.getLogger(__name__)
-
-
-def make_delete_command(
-    repository,
-    config,
-    local_borg_version,
-    delete_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Given a local or remote repository dict, a configuration dict, the local Borg version, the
-    arguments to the delete action as an argparse.Namespace, and global arguments, return a command
-    as a tuple to delete archives from the repository.
-    '''
-    return (
-        (local_path, 'delete')
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + borgmatic.borg.flags.make_flags('dry-run', global_arguments.dry_run)
-        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
-        + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
-        + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
-        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
-        + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives)
-        + (
-            (('--force',) + (('--force',) if delete_arguments.force >= 2 else ()))
-            if delete_arguments.force
-            else ()
-        )
-        # Ignore match_archives and archive_name_format options from configuration, so the user has
-        # to be explicit on the command-line about the archives they want to delete.
-        + borgmatic.borg.flags.make_match_archives_flags(
-            delete_arguments.match_archives or delete_arguments.archive,
-            archive_name_format=None,
-            local_borg_version=local_borg_version,
-            default_archive_name_format='*',
-        )
-        + borgmatic.borg.flags.make_flags_from_arguments(
-            delete_arguments,
-            excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'),
-        )
-        + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
-    )
-
-
-ARCHIVE_RELATED_ARGUMENT_NAMES = (
-    'archive',
-    'match_archives',
-    'first',
-    'last',
-    'oldest',
-    'newest',
-    'older',
-    'newer',
-)
-
-
-def delete_archives(
-    repository,
-    config,
-    local_borg_version,
-    delete_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository dict, a configuration dict, the local Borg version, the
-    arguments to the delete action as an argparse.Namespace, global arguments as an
-    argparse.Namespace, and local and remote Borg paths, delete the selected archives from the
-    repository. If no archives are selected, then delete the entire repository.
-    '''
-    borgmatic.logger.add_custom_log_levels()
-
-    if not any(
-        getattr(delete_arguments, argument_name, None)
-        for argument_name in ARCHIVE_RELATED_ARGUMENT_NAMES
-    ):
-        if borgmatic.borg.feature.available(
-            borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
-        ):
-            logger.warning(
-                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.'
-            )
-
-        repo_delete_arguments = argparse.Namespace(
-            repository=repository['path'],
-            list_archives=delete_arguments.list_archives,
-            force=delete_arguments.force,
-            cache_only=delete_arguments.cache_only,
-            keep_security_info=delete_arguments.keep_security_info,
-        )
-        borgmatic.borg.repo_delete.delete_repository(
-            repository,
-            config,
-            local_borg_version,
-            repo_delete_arguments,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-
-        return
-
-    command = make_delete_command(
-        repository,
-        config,
-        local_borg_version,
-        delete_arguments,
-        global_arguments,
-        local_path,
-        remote_path,
-    )
-
-    borgmatic.execute.execute_command(
-        command,
-        output_log_level=logging.ANSWER,
-        environment=borgmatic.borg.environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )

borgmatic/borg/environment.py

@@ -1,102 +1,39 @@
-import os
-
-import borgmatic.borg.passcommand
-import borgmatic.hooks.credential.parse
-
 OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_base_directory': 'BORG_BASE_DIR',
     'borg_config_directory': 'BORG_CONFIG_DIR',
     'borg_cache_directory': 'BORG_CACHE_DIR',
-    'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
     'borg_security_directory': 'BORG_SECURITY_DIR',
     'borg_keys_directory': 'BORG_KEYS_DIR',
+    'encryption_passcommand': 'BORG_PASSCOMMAND',
+    'encryption_passphrase': 'BORG_PASSPHRASE',
     'ssh_command': 'BORG_RSH',
     'temporary_directory': 'TMPDIR',
 }
 
-DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = {
+DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
     'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
     'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
 }
 
-DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = {
-    'check_i_know_what_i_am_doing': 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING',
-}
 
-
-def make_environment(config):
+def make_environment(storage_config):
     '''
-    Given a borgmatic configuration dict, convert it to a Borg environment variable dict, merge it
-    with a copy of the current environment variables, and return the result.
-
-    Do not reuse this environment across multiple Borg invocations, because it can include
-    references to resources like anonymous pipes for passphrases—which can only be consumed once.
-
-    Here's how native Borg precedence works for a few of the environment variables:
-
-      1. BORG_PASSPHRASE, if set, is used first.
-      2. BORG_PASSCOMMAND is used only if BORG_PASSPHRASE isn't set.
-      3. BORG_PASSPHRASE_FD is used only if neither of the above are set.
-
-    In borgmatic, we want to simulate this precedence order, but there are some additional
-    complications. First, values can come from either configuration or from environment variables
-    set outside borgmatic; configured options should take precedence. Second, when borgmatic gets a
-    passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we
-    want to pass that passphrase to Borg via an anonymous pipe (+ BORG_PASSPHRASE_FD), since that's
-    more secure than using an environment variable (BORG_PASSPHRASE).
+    Given a borgmatic storage configuration dict, return its options converted to a Borg environment
+    variable dict.
     '''
-    environment = dict(os.environ)
+    environment = {}
 
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
-        value = config.get(option_name)
+        value = storage_config.get(option_name)
 
-        if value is not None:
-            environment[environment_variable_name] = str(value)
-
-    if 'encryption_passphrase' in config:
-        environment.pop('BORG_PASSPHRASE', None)
-        environment.pop('BORG_PASSCOMMAND', None)
-
-    if 'encryption_passcommand' in config:
-        environment.pop('BORG_PASSCOMMAND', None)
-
-    passphrase = borgmatic.hooks.credential.parse.resolve_credential(
-        config.get('encryption_passphrase'), config
-    )
-
-    if passphrase is None:
-        passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config)
-
-    # If there's a passphrase (from configuration, from a configured credential, or from a
-    # configured passcommand), send it to Borg via an anonymous pipe.
-    if passphrase is not None:
-        read_file_descriptor, write_file_descriptor = os.pipe()
-        os.write(write_file_descriptor, passphrase.encode('utf-8'))
-        os.close(write_file_descriptor)
-
-        # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the Borg
-        # child process to inherit the file descriptor.
-        os.set_inheritable(read_file_descriptor, True)
-        environment['BORG_PASSPHRASE_FD'] = str(read_file_descriptor)
+        if value:
+            environment[environment_variable_name] = value
 
     for (
         option_name,
         environment_variable_name,
-    ) in DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE.items():
-        if os.environ.get(environment_variable_name) is None:
-            value = config.get(option_name)
-            environment[environment_variable_name] = 'yes' if value else 'no'
-
-    for (
-        option_name,
-        environment_variable_name,
-    ) in DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE.items():
-        value = config.get(option_name)
-        if value is not None:
-            environment[environment_variable_name] = 'YES' if value else 'NO'
-
-    # On Borg 1.4.0a1+, take advantage of more specific exit codes. No effect on
-    # older versions of Borg.
-    environment['BORG_EXIT_CODES'] = 'modern'
+    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name, False)
+        environment[environment_variable_name] = 'yes' if value else 'no'
 
     return environment

borgmatic/borg/export_key.py

@@ -1,74 +0,0 @@
-import logging
-import os
-
-import borgmatic.config.paths
-import borgmatic.logger
-from borgmatic.borg import environment, flags
-from borgmatic.execute import DO_NOT_CAPTURE, execute_command
-
-logger = logging.getLogger(__name__)
-
-
-def export_key(
-    repository_path,
-    config,
-    local_borg_version,
-    export_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, export
-    arguments, and optional local and remote Borg paths, export the repository key to the
-    destination path indicated in the export arguments.
-
-    If the destination path is empty or "-", then print the key to stdout instead of to a file.
-
-    Raise FileExistsError if a path is given but it already exists on disk.
-    '''
-    borgmatic.logger.add_custom_log_levels()
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    if export_arguments.path and export_arguments.path != '-':
-        if os.path.exists(os.path.join(working_directory or '', export_arguments.path)):
-            raise FileExistsError(
-                f'Destination path {export_arguments.path} already exists. Aborting.'
-            )
-
-        output_file = None
-    else:
-        output_file = DO_NOT_CAPTURE
-
-    full_command = (
-        (local_path, 'key', 'export')
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + flags.make_flags('paper', export_arguments.paper)
-        + flags.make_flags('qr-html', export_arguments.qr_html)
-        + flags.make_repository_flags(
-            repository_path,
-            local_borg_version,
-        )
-        + ((export_arguments.path,) if output_file is None else ())
-    )
-
-    if global_arguments.dry_run:
-        logger.info('Skipping key export (dry run)')
-        return
-
-    execute_command(
-        full_command,
-        output_file=output_file,
-        output_log_level=logging.ANSWER,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )

borgmatic/borg/export_tar.py

@@ -1,8 +1,7 @@
 import logging
+import os
 
-import borgmatic.config.paths
-import borgmatic.logger
-from borgmatic.borg import environment, flags
+from borgmatic.borg import environment
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
@@ -10,68 +9,58 @@ logger = logging.getLogger(__name__)
 
 def export_tar_archive(
     dry_run,
-    repository_path,
+    repository,
     archive,
     paths,
     destination_path,
-    config,
-    local_borg_version,
-    global_arguments,
+    storage_config,
     local_path='borg',
     remote_path=None,
     tar_filter=None,
-    list_files=False,
+    files=False,
     strip_components=None,
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
-    export from the archive, a destination path to export to, a configuration dict, the local Borg
-    version, optional local and remote Borg paths, an optional filter program, whether to include
-    per-file details, and an optional number of path components to strip, export the archive into
-    the given destination path as a tar-formatted file.
+    export from the archive, a destination path to export to, a storage configuration dict, optional
+    local and remote Borg paths, an optional filter program, whether to include per-file details,
+    and an optional number of path components to strip, export the archive into the given
+    destination path as a tar-formatted file.
 
     If the destination path is "-", then stream the output to stdout instead of to a file.
     '''
-    borgmatic.logger.add_custom_log_levels()
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
         (local_path, 'export-tar')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--list',) if list_files else ())
+        + (('--list',) if files else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (('--tar-filter', tar_filter) if tar_filter else ())
         + (('--strip-components', str(strip_components)) if strip_components else ())
-        + flags.make_repository_archive_flags(
-            repository_path,
-            archive,
-            local_borg_version,
-        )
+        + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
         + (destination_path,)
         + (tuple(paths) if paths else ())
     )
 
-    if list_files:
-        output_log_level = logging.ANSWER
+    if files and logger.getEffectiveLevel() == logging.WARNING:
+        output_log_level = logging.WARNING
     else:
         output_log_level = logging.INFO
 
     if dry_run:
-        logging.info('Skipping export to tar file (dry run)')
+        logging.info('{}: Skipping export to tar file (dry run)'.format(repository))
         return
 
     execute_command(
         full_command,
         output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
         output_log_level=output_log_level,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
+        extra_environment=environment.make_environment(storage_config),
     )

borgmatic/borg/extract.py

@@ -2,66 +2,65 @@ import logging
 import os
 import subprocess
 
-import borgmatic.config.paths
-import borgmatic.config.validate
-from borgmatic.borg import environment, feature, flags, repo_list
+from borgmatic.borg import environment, feature
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
 
 def extract_last_archive_dry_run(
-    config,
-    local_borg_version,
-    global_arguments,
-    repository_path,
-    lock_wait=None,
-    local_path='borg',
-    remote_path=None,
+    storage_config, repository, lock_wait=None, local_path='borg', remote_path=None
 ):
     '''
     Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
     dry-run.
     '''
+    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
     verbosity_flags = ()
     if logger.isEnabledFor(logging.DEBUG):
         verbosity_flags = ('--debug', '--show-rc')
     elif logger.isEnabledFor(logging.INFO):
         verbosity_flags = ('--info',)
 
+    full_list_command = (
+        (local_path, 'list', '--short')
+        + remote_path_flags
+        + lock_wait_flags
+        + verbosity_flags
+        + (repository,)
+    )
+
+    borg_environment = environment.make_environment(storage_config)
+
+    list_output = execute_command(
+        full_list_command,
+        output_log_level=None,
+        borg_local_path=local_path,
+        extra_environment=borg_environment,
+    )
+
     try:
-        last_archive_name = repo_list.resolve_archive_name(
-            repository_path,
-            'latest',
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-    except ValueError:
-        logger.warning('No archives found. Skipping extract consistency check.')
+        last_archive_name = list_output.strip().splitlines()[-1]
+    except IndexError:
         return
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
     full_extract_command = (
         (local_path, 'extract', '--dry-run')
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + remote_path_flags
+        + lock_wait_flags
         + verbosity_flags
         + list_flag
-        + flags.make_repository_archive_flags(
-            repository_path, last_archive_name, local_borg_version
+        + (
+            '{repository}::{last_archive_name}'.format(
+                repository=repository, last_archive_name=last_archive_name
+            ),
         )
     )
 
     execute_command(
-        full_extract_command,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
+        full_extract_command, working_directory=None, extra_environment=borg_environment
     )
 
 
@@ -70,9 +69,9 @@ def extract_archive(
     repository,
     archive,
     paths,
-    config,
+    location_config,
+    storage_config,
     local_borg_version,
-    global_arguments,
     local_path='borg',
     remote_path=None,
     destination_path=None,
@@ -82,47 +81,29 @@ def extract_archive(
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
-    restore from the archive, the local Borg version string, an argparse.Namespace of global
-    arguments, a configuration dict, optional local and remote Borg paths, and an optional
-    destination path to extract to, extract the archive into the current directory.
+    restore from the archive, the local Borg version string, location/storage configuration dicts,
+    optional local and remote Borg paths, and an optional destination path to extract to, extract
+    the archive into the current directory.
 
     If extract to stdout is True, then start the extraction streaming to stdout, and return that
     extract process as an instance of subprocess.Popen.
     '''
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
 
     if progress and extract_to_stdout:
         raise ValueError('progress and extract_to_stdout cannot both be set')
 
     if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version):
-        numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
+        numeric_ids_flags = ('--numeric-ids',) if location_config.get('numeric_owner') else ()
     else:
-        numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else ()
-
-    if strip_components == 'all':
-        if not paths:
-            raise ValueError('The --strip-components flag with "all" requires at least one --path')
-
-        # Calculate the maximum number of leading path components of the given paths. "if piece"
-        # ignores empty path components, e.g. those resulting from a leading slash. And the "- 1"
-        # is so this doesn't count the final path component, e.g. the filename itself.
-        strip_components = max(
-            0,
-            *(
-                len(tuple(piece for piece in path.split(os.path.sep) if piece)) - 1
-                for path in paths
-            ),
-        )
-
-    working_directory = borgmatic.config.paths.get_working_directory(config)
+        numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_owner') else ()
 
     full_command = (
         (local_path, 'extract')
         + (('--remote-path', remote_path) if remote_path else ())
         + numeric_ids_flags
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
@@ -130,21 +111,11 @@ def extract_archive(
         + (('--strip-components', str(strip_components)) if strip_components else ())
         + (('--progress',) if progress else ())
         + (('--stdout',) if extract_to_stdout else ())
-        + flags.make_repository_archive_flags(
-            # Make the repository path absolute so the destination directory used below via changing
-            # the working directory doesn't prevent Borg from finding the repo. But also apply the
-            # user's configured working directory (if any) to the repo path.
-            borgmatic.config.validate.normalize_repository_path(repository, working_directory),
-            archive,
-            local_borg_version,
-        )
+        + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
         + (tuple(paths) if paths else ())
     )
 
-    borg_exit_codes = config.get('borg_exit_codes')
-    full_destination_path = (
-        os.path.join(working_directory or '', destination_path) if destination_path else None
-    )
+    borg_environment = environment.make_environment(storage_config)
 
     # The progress output isn't compatible with captured and logged output, as progress messes with
     # the terminal directly.
@@ -152,10 +123,8 @@ def extract_archive(
         return execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            environment=environment.make_environment(config),
-            working_directory=full_destination_path,
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+            working_directory=destination_path,
+            extra_environment=borg_environment,
         )
         return None
 
@@ -163,19 +132,13 @@ def extract_archive(
         return execute_command(
             full_command,
             output_file=subprocess.PIPE,
+            working_directory=destination_path,
             run_to_completion=False,
-            environment=environment.make_environment(config),
-            working_directory=full_destination_path,
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+            extra_environment=borg_environment,
         )
 
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
     execute_command(
-        full_command,
-        environment=environment.make_environment(config),
-        working_directory=full_destination_path,
-        borg_local_path=local_path,
-        borg_exit_codes=borg_exit_codes,
+        full_command, working_directory=destination_path, extra_environment=borg_environment
     )

borgmatic/borg/feature.py

@@ -1,6 +1,6 @@
 from enum import Enum
 
-from packaging.version import parse
+from pkg_resources import parse_version
 
 
 class Feature(Enum):
@@ -9,32 +9,14 @@ class Feature(Enum):
     NOFLAGS = 3
     NUMERIC_IDS = 4
     UPLOAD_RATELIMIT = 5
-    SEPARATE_REPOSITORY_ARCHIVE = 6
-    REPO_CREATE = 7
-    REPO_LIST = 8
-    REPO_INFO = 9
-    REPO_DELETE = 10
-    MATCH_ARCHIVES = 11
-    EXCLUDED_FILES_MINUS = 12
-    ARCHIVE_SERIES = 13
-    NO_PRUNE_STATS = 14
 
 
 FEATURE_TO_MINIMUM_BORG_VERSION = {
-    Feature.COMPACT: parse('1.2.0a2'),  # borg compact
-    Feature.ATIME: parse('1.2.0a7'),  # borg create --atime
-    Feature.NOFLAGS: parse('1.2.0a8'),  # borg create --noflags
-    Feature.NUMERIC_IDS: parse('1.2.0b3'),  # borg create/extract/mount --numeric-ids
-    Feature.UPLOAD_RATELIMIT: parse('1.2.0b3'),  # borg create --upload-ratelimit
-    Feature.SEPARATE_REPOSITORY_ARCHIVE: parse('2.0.0a2'),  # --repo with separate archive
-    Feature.REPO_CREATE: parse('2.0.0a2'),  # borg repo-create
-    Feature.REPO_LIST: parse('2.0.0a2'),  # borg repo-list
-    Feature.REPO_INFO: parse('2.0.0a2'),  # borg repo-info
-    Feature.REPO_DELETE: parse('2.0.0a2'),  # borg repo-delete
-    Feature.MATCH_ARCHIVES: parse('2.0.0b3'),  # borg --match-archives
-    Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'),  # --list --filter uses "-" for excludes
-    Feature.ARCHIVE_SERIES: parse('2.0.0b11'),  # identically named archives form a series
-    Feature.NO_PRUNE_STATS: parse('2.0.0b10'),  # prune --stats is not available
+    Feature.COMPACT: parse_version('1.2.0a2'),  # borg compact
+    Feature.ATIME: parse_version('1.2.0a7'),  # borg create --atime
+    Feature.NOFLAGS: parse_version('1.2.0a8'),  # borg create --noflags
+    Feature.NUMERIC_IDS: parse_version('1.2.0b3'),  # borg create/extract/mount --numeric-ids
+    Feature.UPLOAD_RATELIMIT: parse_version('1.2.0b3'),  # borg create --upload-ratelimit
 }
 
 
@@ -43,4 +25,4 @@ def available(feature, borg_version):
     Given a Borg Feature constant and a Borg version string, return whether that feature is
     available in that version of Borg.
     '''
-    return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse(borg_version)
+    return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse_version(borg_version)

borgmatic/borg/flags.py

@@ -1,11 +1,4 @@
 import itertools
-import json
-import logging
-import re
-
-from borgmatic.borg import feature
-
-logger = logging.getLogger(__name__)
 
 
 def make_flags(name, value):
@@ -15,7 +8,7 @@ def make_flags(name, value):
     if not value:
         return ()
 
-    flag = f"--{name.replace('_', '-')}"
+    flag = '--{}'.format(name.replace('_', '-'))
 
     if value is True:
         return (flag,)
@@ -36,164 +29,3 @@ def make_flags_from_arguments(arguments, excludes=()):
             if name not in excludes and not name.startswith('_')
         )
     )
-
-
-def make_repository_flags(repository_path, local_borg_version):
-    '''
-    Given the path of a Borg repository and the local Borg version, return Borg-version-appropriate
-    command-line flags (as a tuple) for selecting that repository.
-    '''
-    return (
-        ('--repo',)
-        if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version)
-        else ()
-    ) + (repository_path,)
-
-
-ARCHIVE_HASH_PATTERN = re.compile('[0-9a-fA-F]{8,}$')
-
-
-def make_repository_archive_flags(repository_path, archive, local_borg_version):
-    '''
-    Given the path of a Borg repository, an archive name or pattern, and the local Borg version,
-    return Borg-version-appropriate command-line flags (as a tuple) for selecting that repository
-    and archive.
-    '''
-    return (
-        (
-            '--repo',
-            repository_path,
-            (
-                f'aid:{archive}'
-                if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
-                and ARCHIVE_HASH_PATTERN.match(archive)
-                and not archive.startswith('aid:')
-                else archive
-            ),
-        )
-        if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version)
-        else (f'{repository_path}::{archive}',)
-    )
-
-
-DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
-DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'  # noqa: FS003
-
-
-def get_default_archive_name_format(local_borg_version):
-    '''
-    Given the local Borg version, return the corresponding default archive name format.
-    '''
-    if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version):
-        return DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES
-
-    return DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES
-
-
-def make_match_archives_flags(
-    match_archives,
-    archive_name_format,
-    local_borg_version,
-    default_archive_name_format=None,
-):
-    '''
-    Return match archives flags based on the given match archives value, if any. If it isn't set,
-    return match archives flags to match archives created with the given (or default) archive name
-    format. This is done by replacing certain archive name format placeholders for ephemeral data
-    (like "{now}") with globs.
-    '''
-    if match_archives:
-        if match_archives in {'*', 're:.*', 'sh:*'}:
-            return ()
-
-        if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
-            if (
-                feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
-                and ARCHIVE_HASH_PATTERN.match(match_archives)
-                and not match_archives.startswith('aid:')
-            ):
-                return ('--match-archives', f'aid:{match_archives}')
-
-            return ('--match-archives', match_archives)
-        else:
-            return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
-
-    derived_match_archives = re.sub(
-        r'\{(now|utcnow|pid)([:%\w\.-]*)\}',
-        '*',
-        archive_name_format
-        or default_archive_name_format
-        or get_default_archive_name_format(local_borg_version),
-    )
-
-    if derived_match_archives == '*':
-        return ()
-
-    if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
-        return ('--match-archives', f'sh:{derived_match_archives}')
-    else:
-        return ('--glob-archives', f'{derived_match_archives}')
-
-
-def warn_for_aggressive_archive_flags(json_command, json_output):
-    '''
-    Given a JSON archives command and the resulting JSON string output from running it, parse the
-    JSON and warn if the command used an archive flag but the output indicates zero archives were
-    found.
-    '''
-    archive_flags_used = {'--glob-archives', '--match-archives'}.intersection(set(json_command))
-
-    if not archive_flags_used:
-        return
-
-    try:
-        if len(json.loads(json_output)['archives']) == 0:
-            logger.warning('An archive filter was applied, but no matching archives were found.')
-            logger.warning(
-                'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.'
-            )
-    except json.JSONDecodeError as error:
-        logger.debug(f'Cannot parse JSON output from archive command: {error}')
-    except (TypeError, KeyError):
-        logger.debug('Cannot parse JSON output from archive command: No "archives" key found')
-
-
-def omit_flag(arguments, flag):
-    '''
-    Given a sequence of Borg command-line arguments, return them with the given (valueless) flag
-    omitted. For instance, if the flag is "--flag" and arguments is:
-
-        ('borg', 'create', '--flag', '--other-flag')
-
-    ... then return:
-
-        ('borg', 'create', '--other-flag')
-    '''
-    return tuple(argument for argument in arguments if argument != flag)
-
-
-def omit_flag_and_value(arguments, flag):
-    '''
-    Given a sequence of Borg command-line arguments, return them with the given flag and its
-    corresponding value omitted. For instance, if the flag is "--flag" and arguments is:
-
-        ('borg', 'create', '--flag', 'value', '--other-flag')
-
-    ... or:
-
-        ('borg', 'create', '--flag=value', '--other-flag')
-
-    ... then return:
-
-        ('borg', 'create', '--other-flag')
-    '''
-    # This works by zipping together a list of overlapping pairwise arguments. E.g., ('one', 'two',
-    # 'three', 'four') becomes ((None, 'one'), ('one, 'two'), ('two', 'three'), ('three', 'four')).
-    # This makes it easy to "look back" at the previous arguments so we can exclude both a flag and
-    # its value.
-    return tuple(
-        argument
-        for (previous_argument, argument) in zip((None,) + arguments, arguments)
-        if flag not in (previous_argument, argument)
-        if not argument.startswith(f'{flag}=')
-    )

borgmatic/borg/import_key.py

@@ -1,70 +0,0 @@
-import logging
-import os
-
-import borgmatic.config.paths
-import borgmatic.logger
-from borgmatic.borg import environment, flags
-from borgmatic.execute import DO_NOT_CAPTURE, execute_command
-
-logger = logging.getLogger(__name__)
-
-
-def import_key(
-    repository_path,
-    config,
-    local_borg_version,
-    import_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, import
-    arguments, and optional local and remote Borg paths, import the repository key from the
-    path indicated in the import arguments.
-
-    If the path is empty or "-", then read the key from stdin.
-
-    Raise ValueError if the path is given and it does not exist.
-    '''
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    if import_arguments.path and import_arguments.path != '-':
-        if not os.path.exists(os.path.join(working_directory or '', import_arguments.path)):
-            raise ValueError(f'Path {import_arguments.path} does not exist. Aborting.')
-
-        input_file = None
-    else:
-        input_file = DO_NOT_CAPTURE
-
-    full_command = (
-        (local_path, 'key', 'import')
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + flags.make_flags('paper', import_arguments.paper)
-        + flags.make_repository_flags(
-            repository_path,
-            local_borg_version,
-        )
-        + ((import_arguments.path,) if input_file is None else ())
-    )
-
-    if global_arguments.dry_run:
-        logger.info('Skipping key import (dry run)')
-        return
-
-    execute_command(
-        full_command,
-        input_file=input_file,
-        output_log_level=logging.INFO,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )

borgmatic/borg/info.py

@@ -1,29 +1,23 @@
-import argparse
 import logging
 
-import borgmatic.config.paths
-import borgmatic.logger
-from borgmatic.borg import environment, feature, flags
-from borgmatic.execute import execute_command, execute_command_and_capture_output
+from borgmatic.borg import environment
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def make_info_command(
-    repository_path,
-    config,
-    local_borg_version,
-    info_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
+def display_archives_info(
+    repository, storage_config, info_arguments, local_path='borg', remote_path=None
 ):
     '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, the
-    arguments to the info action as an argparse.Namespace, and global arguments, return a command
-    as a tuple to display summary information for archives in the repository.
+    Given a local or remote repository path, a storage config dict, and the arguments to the info
+    action, display summary information for Borg archives in the repository or return JSON summary
+    information.
     '''
-    return (
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
         (local_path, 'info')
         + (
             ('--info',)
@@ -35,89 +29,19 @@ def make_info_command(
             if logger.isEnabledFor(logging.DEBUG) and not info_arguments.json
             else ()
         )
-        + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', global_arguments.log_json)
-        + flags.make_flags('lock-wait', config.get('lock_wait'))
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(info_arguments, excludes=('repository', 'archive'))
         + (
-            (
-                flags.make_flags('match-archives', f'sh:{info_arguments.prefix}*')
-                if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
-                else flags.make_flags('glob-archives', f'{info_arguments.prefix}*')
-            )
-            if info_arguments.prefix
-            else (
-                flags.make_match_archives_flags(
-                    info_arguments.match_archives
-                    or info_arguments.archive
-                    or config.get('match_archives'),
-                    config.get('archive_name_format'),
-                    local_borg_version,
-                )
-            )
+            '::'.join((repository, info_arguments.archive))
+            if info_arguments.archive
+            else repository,
         )
-        + flags.make_flags_from_arguments(
-            info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives')
-        )
-        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-
-def display_archives_info(
-    repository_path,
-    config,
-    local_borg_version,
-    info_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, the
-    arguments to the info action as an argparse.Namespace, and global arguments, display summary
-    information for Borg archives in the repository or return JSON summary information.
-    '''
-    borgmatic.logger.add_custom_log_levels()
-
-    main_command = make_info_command(
-        repository_path,
-        config,
-        local_borg_version,
-        info_arguments,
-        global_arguments,
-        local_path,
-        remote_path,
-    )
-    json_command = make_info_command(
-        repository_path,
-        config,
-        local_borg_version,
-        argparse.Namespace(**dict(info_arguments.__dict__, json=True)),
-        global_arguments,
-        local_path,
-        remote_path,
-    )
-    borg_exit_codes = config.get('borg_exit_codes')
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    json_info = execute_command_and_capture_output(
-        json_command,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
+    return execute_command(
+        full_command,
+        output_log_level=None if info_arguments.json else logging.WARNING,
         borg_local_path=local_path,
-        borg_exit_codes=borg_exit_codes,
-    )
-
-    if info_arguments.json:
-        return json_info
-
-    flags.warn_for_aggressive_archive_flags(json_command, json_info)
-
-    execute_command(
-        main_command,
-        output_log_level=logging.ANSWER,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=borg_exit_codes,
+        extra_environment=environment.make_environment(storage_config),
     )

borgmatic/borg/init.py

@@ -0,0 +1,62 @@
+import argparse
+import logging
+import subprocess
+
+from borgmatic.borg import environment, info
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
+
+logger = logging.getLogger(__name__)
+
+
+INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
+
+
+def initialize_repository(
+    repository,
+    storage_config,
+    encryption_mode,
+    append_only=None,
+    storage_quota=None,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a storage configuration dict, a Borg encryption mode,
+    whether the repository should be append-only, and the storage quota to use, initialize the
+    repository. If the repository already exists, then log and skip initialization.
+    '''
+    try:
+        info.display_archives_info(
+            repository,
+            storage_config,
+            argparse.Namespace(json=True, archive=None),
+            local_path,
+            remote_path,
+        )
+        logger.info('Repository already exists. Skipping initialization.')
+        return
+    except subprocess.CalledProcessError as error:
+        if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
+            raise
+
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('init', '')
+
+    init_command = (
+        (local_path, 'init')
+        + (('--encryption', encryption_mode) if encryption_mode else ())
+        + (('--append-only',) if append_only else ())
+        + (('--storage-quota', storage_quota) if storage_quota else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (repository,)
+    )
+
+    # Do not capture output here, so as to support interactive prompts.
+    execute_command(
+        init_command,
+        output_file=DO_NOT_CAPTURE,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
+    )

borgmatic/borg/list.py

@@ -1,39 +1,66 @@
-import argparse
 import copy
 import logging
 import re
 
-import borgmatic.config.paths
-import borgmatic.logger
-from borgmatic.borg import environment, feature, flags, repo_list
-from borgmatic.execute import execute_command, execute_command_and_capture_output
+from borgmatic.borg import environment
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST = ('prefix', 'match_archives', 'sort_by', 'first', 'last')
-MAKE_FLAGS_EXCLUDES = (
-    'repository',
-    'archive',
-    'paths',
-    'find_paths',
-) + ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST
+def resolve_archive_name(repository, archive, storage_config, local_path='borg', remote_path=None):
+    '''
+    Given a local or remote repository path, an archive name, a storage config dict, a local Borg
+    path, and a remote Borg path, simply return the archive name. But if the archive name is
+    "latest", then instead introspect the repository for the latest archive and return its name.
+
+    Raise ValueError if "latest" is given but there are no archives in the repository.
+    '''
+    if archive != "latest":
+        return archive
+
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'list')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags('last', 1)
+        + ('--short', repository)
+    )
+
+    output = execute_command(
+        full_command,
+        output_log_level=None,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
+    )
+    try:
+        latest_archive = output.strip().splitlines()[-1]
+    except IndexError:
+        raise ValueError('No archives found in the repository')
+
+    logger.debug('{}: Latest archive is {}'.format(repository, latest_archive))
+
+    return latest_archive
+
+
+MAKE_FLAGS_EXCLUDES = ('repository', 'archive', 'successful', 'paths', 'find_paths')
 
 
 def make_list_command(
-    repository_path,
-    config,
-    local_borg_version,
-    list_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
+    repository, storage_config, list_arguments, local_path='borg', remote_path=None
 ):
     '''
-    Given a local or remote repository path, a configuration dict, the arguments to the list action,
-    and local and remote Borg paths, return a command as a tuple to list archives or paths within an
-    archive.
+    Given a local or remote repository path, a storage config dict, the arguments to the list
+    action, and local and remote Borg paths, return a command as a tuple to list archives or paths
+    within an archive.
     '''
+    lock_wait = storage_config.get('lock_wait', None)
+
     return (
         (local_path, 'list')
         + (
@@ -46,17 +73,13 @@ def make_list_command(
             if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json
             else ()
         )
-        + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', global_arguments.log_json)
-        + flags.make_flags('lock-wait', config.get('lock_wait'))
-        + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES,)
         + (
-            flags.make_repository_archive_flags(
-                repository_path, list_arguments.archive, local_borg_version
-            )
+            ('::'.join((repository, list_arguments.archive)),)
             if list_arguments.archive
-            else flags.make_repository_flags(repository_path, local_borg_version)
+            else (repository,)
         )
         + (tuple(list_arguments.paths) if list_arguments.paths else ())
     )
@@ -79,186 +102,69 @@ def make_find_paths(find_paths):
         return ()
 
     return tuple(
-        (
-            find_path
-            if re.compile(r'([-!+RrPp] )|(\w\w:)').match(find_path)
-            else f'sh:**/*{find_path}*/**'
-        )
+        find_path
+        if re.compile(r'([-!+RrPp] )|(\w\w:)').match(find_path)
+        else f'sh:**/*{find_path}*/**'
         for find_path in find_paths
     )
 
 
-def capture_archive_listing(
-    repository_path,
-    archive,
-    config,
-    local_borg_version,
-    global_arguments,
-    list_paths=None,
-    path_format=None,
-    local_path='borg',
-    remote_path=None,
-):
+def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
     '''
-    Given a local or remote repository path, an archive name, a configuration
-    dict, the local Borg version, global arguments as an argparse.Namespace,
-    the archive paths (or Borg patterns) in which to list files, the Borg path
-    format to use for the output, and local and remote Borg paths, capture the
-    output of listing that archive and return it as a list of file paths.
+    Given a local or remote repository path, a storage config dict, the arguments to the list
+    action, and local and remote Borg paths, display the output of listing Borg archives in the
+    repository or return JSON output. Or, if an archive name is given, list the files in that
+    archive. Or, if list_arguments.find_paths are given, list the files by searching across multiple
+    archives.
     '''
-    return tuple(
-        execute_command_and_capture_output(
-            make_list_command(
-                repository_path,
-                config,
-                local_borg_version,
-                argparse.Namespace(
-                    repository=repository_path,
-                    archive=archive,
-                    paths=[path for path in list_paths] if list_paths else None,
-                    find_paths=None,
-                    json=None,
-                    format=path_format or '{path}{NUL}',  # noqa: FS003
-                ),
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            environment=environment.make_environment(config),
-            working_directory=borgmatic.config.paths.get_working_directory(config),
-            borg_local_path=local_path,
-            borg_exit_codes=config.get('borg_exit_codes'),
-        )
-        .strip('\0')
-        .split('\0')
-    )
-
-
-def list_archive(
-    repository_path,
-    config,
-    local_borg_version,
-    list_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, the
-    arguments to the list action as an argparse.Namespace, global arguments as an
-    argparse.Namespace, and local and remote Borg paths, display the output of listing the files of
-    a Borg archive (or return JSON output). If list_arguments.find_paths are given, list the files
-    by searching across multiple archives. If neither find_paths nor archive name are given, instead
-    list the archives in the given repository.
-    '''
-    borgmatic.logger.add_custom_log_levels()
-
-    if not list_arguments.archive and not list_arguments.find_paths:
-        if feature.available(feature.Feature.REPO_LIST, local_borg_version):
-            logger.warning(
-                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.'
-            )
-
-        repo_list_arguments = argparse.Namespace(
-            repository=repository_path,
-            short=list_arguments.short,
-            format=list_arguments.format,
-            json=list_arguments.json,
-            prefix=list_arguments.prefix,
-            match_archives=list_arguments.match_archives,
-            sort_by=list_arguments.sort_by,
-            first=list_arguments.first,
-            last=list_arguments.last,
-        )
-        return repo_list.list_repository(
-            repository_path,
-            config,
-            local_borg_version,
-            repo_list_arguments,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-
-    if list_arguments.archive:
-        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
-            if getattr(list_arguments, name, None):
-                logger.warning(
-                    f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag."
-                )
-
-    if list_arguments.json:
-        raise ValueError(
-            'The --json flag on the list action is not supported when using the --archive/--find flags.'
-        )
-
-    borg_exit_codes = config.get('borg_exit_codes')
+    borg_environment = environment.make_environment(storage_config)
 
     # If there are any paths to find (and there's not a single archive already selected), start by
     # getting a list of archives to search.
     if list_arguments.find_paths and not list_arguments.archive:
-        repo_list_arguments = argparse.Namespace(
-            repository=repository_path,
-            short=True,
-            format=None,
-            json=None,
-            prefix=list_arguments.prefix,
-            match_archives=list_arguments.match_archives,
-            sort_by=list_arguments.sort_by,
-            first=list_arguments.first,
-            last=list_arguments.last,
-        )
+        repository_arguments = copy.copy(list_arguments)
+        repository_arguments.archive = None
+        repository_arguments.json = False
+        repository_arguments.format = None
 
         # Ask Borg to list archives. Capture its output for use below.
         archive_lines = tuple(
-            execute_command_and_capture_output(
-                repo_list.make_repo_list_command(
-                    repository_path,
-                    config,
-                    local_borg_version,
-                    repo_list_arguments,
-                    global_arguments,
-                    local_path,
-                    remote_path,
+            execute_command(
+                make_list_command(
+                    repository, storage_config, repository_arguments, local_path, remote_path
                 ),
-                environment=environment.make_environment(config),
-                working_directory=borgmatic.config.paths.get_working_directory(config),
+                output_log_level=None,
                 borg_local_path=local_path,
-                borg_exit_codes=borg_exit_codes,
+                extra_environment=borg_environment,
             )
             .strip('\n')
-            .splitlines()
+            .split('\n')
         )
     else:
         archive_lines = (list_arguments.archive,)
 
     # For each archive listed by Borg, run list on the contents of that archive.
-    for archive in archive_lines:
-        logger.answer(f'Listing archive {archive}')
+    for archive_line in archive_lines:
+        try:
+            archive = archive_line.split()[0]
+        except (AttributeError, IndexError):
+            archive = None
+
+        if archive:
+            logger.warning(archive_line)
 
         archive_arguments = copy.copy(list_arguments)
         archive_arguments.archive = archive
-
-        # This list call is to show the files in a single archive, not list multiple archives. So
-        # blank out any archive filtering flags. They'll break anyway in Borg 2.
-        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
-            setattr(archive_arguments, name, None)
-
         main_command = make_list_command(
-            repository_path,
-            config,
-            local_borg_version,
-            archive_arguments,
-            global_arguments,
-            local_path,
-            remote_path,
+            repository, storage_config, archive_arguments, local_path, remote_path
         ) + make_find_paths(list_arguments.find_paths)
 
-        execute_command(
+        output = execute_command(
             main_command,
-            output_log_level=logging.ANSWER,
-            environment=environment.make_environment(config),
-            working_directory=borgmatic.config.paths.get_working_directory(config),
+            output_log_level=None if list_arguments.json else logging.WARNING,
             borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+            extra_environment=borg_environment,
         )
+
+        if list_arguments.json:
+            return output

borgmatic/borg/mount.py

@@ -1,82 +1,54 @@
 import logging
 
-import borgmatic.config.paths
-from borgmatic.borg import environment, feature, flags
+from borgmatic.borg import environment
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
 
 def mount_archive(
-    repository_path,
+    repository,
     archive,
-    mount_arguments,
-    config,
-    local_borg_version,
-    global_arguments,
+    mount_point,
+    paths,
+    foreground,
+    options,
+    storage_config,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, an optional archive name, a filesystem mount point,
     zero or more paths to mount from the archive, extra Borg mount options, a storage configuration
-    dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional
-    local and remote Borg paths, mount the archive onto the mount point.
+    dict, and optional local and remote Borg paths, mount the archive onto the mount point.
     '''
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
         (local_path, 'mount')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + flags.make_flags_from_arguments(
-            mount_arguments,
-            excludes=('repository', 'archive', 'mount_point', 'paths', 'options'),
-        )
-        + (('-o', mount_arguments.options) if mount_arguments.options else ())
-        + (
-            (
-                flags.make_repository_flags(repository_path, local_borg_version)
-                + (
-                    ('--match-archives', archive)
-                    if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
-                    else ('--glob-archives', archive)
-                )
-            )
-            if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version)
-            else (
-                flags.make_repository_archive_flags(repository_path, archive, local_borg_version)
-                if archive
-                else flags.make_repository_flags(repository_path, local_borg_version)
-            )
-        )
-        + (mount_arguments.mount_point,)
-        + (tuple(mount_arguments.paths) if mount_arguments.paths else ())
+        + (('--foreground',) if foreground else ())
+        + (('-o', options) if options else ())
+        + (('::'.join((repository, archive)),) if archive else (repository,))
+        + (mount_point,)
+        + (tuple(paths) if paths else ())
     )
 
-    working_directory = borgmatic.config.paths.get_working_directory(config)
+    borg_environment = environment.make_environment(storage_config)
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
-    if mount_arguments.foreground:
+    if foreground:
         execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            environment=environment.make_environment(config),
-            working_directory=working_directory,
             borg_local_path=local_path,
-            borg_exit_codes=config.get('borg_exit_codes'),
+            extra_environment=borg_environment,
         )
         return
 
-    execute_command(
-        full_command,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )
+    execute_command(full_command, borg_local_path=local_path, extra_environment=borg_environment)

borgmatic/borg/passcommand.py

@@ -1,40 +0,0 @@
-import functools
-import logging
-import shlex
-
-import borgmatic.config.paths
-import borgmatic.execute
-
-logger = logging.getLogger(__name__)
-
-
-@functools.cache
-def run_passcommand(passcommand, working_directory):
-    '''
-    Run the given passcommand using the given working directory and return the passphrase produced
-    by the command.
-
-    Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
-    per borgmatic invocation.
-    '''
-    return borgmatic.execute.execute_command_and_capture_output(
-        shlex.split(passcommand),
-        working_directory=working_directory,
-    )
-
-
-def get_passphrase_from_passcommand(config):
-    '''
-    Given the configuration dict, call the configured passcommand to produce and return an
-    encryption passphrase. In effect, we're doing an end-run around Borg by invoking its passcommand
-    ourselves. This allows us to pass the resulting passphrase to multiple different Borg
-    invocations without the user having to be prompted multiple times.
-
-    If no passcommand is configured, then return None.
-    '''
-    passcommand = config.get('encryption_passcommand')
-
-    if not passcommand:
-        return None
-
-    return run_passcommand(passcommand, borgmatic.config.paths.get_working_directory(config))

borgmatic/borg/pattern.py

@@ -1,50 +0,0 @@
-import collections
-import enum
-
-
-# See https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns
-class Pattern_type(enum.Enum):
-    ROOT = 'R'  # A ROOT pattern always has a NONE pattern style.
-    PATTERN_STYLE = 'P'
-    EXCLUDE = '-'
-    NO_RECURSE = '!'
-    INCLUDE = '+'
-
-
-class Pattern_style(enum.Enum):
-    NONE = ''
-    FNMATCH = 'fm'
-    SHELL = 'sh'
-    REGULAR_EXPRESSION = 're'
-    PATH_PREFIX = 'pp'
-    PATH_FULL_MATCH = 'pf'
-
-
-class Pattern_source(enum.Enum):
-    '''
-    Where the pattern came from within borgmatic. This is important because certain use cases (like
-    filesystem snapshotting) only want to consider patterns that the user actually put in a
-    configuration file and not patterns from other sources.
-    '''
-
-    # The pattern is from a borgmatic configuration option, e.g. listed in "source_directories".
-    CONFIG = 'config'
-
-    # The pattern is generated internally within borgmatic, e.g. for special file excludes.
-    INTERNAL = 'internal'
-
-    # The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump
-    # directory.
-    HOOK = 'hook'
-
-
-Pattern = collections.namedtuple(
-    'Pattern',
-    ('path', 'type', 'style', 'device', 'source'),
-    defaults=(
-        Pattern_type.ROOT,
-        Pattern_style.NONE,
-        None,
-        Pattern_source.HOOK,
-    ),
-)

borgmatic/borg/prune.py

@@ -1,18 +1,15 @@
 import logging
 
-import borgmatic.config.paths
-import borgmatic.logger
-from borgmatic.borg import environment, feature, flags
+from borgmatic.borg import environment
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def make_prune_flags(config, prune_arguments, local_borg_version):
+def _make_prune_flags(retention_config):
     '''
-    Given a configuration dict mapping from option name to value, prune arguments as an
-    argparse.Namespace instance, and the local Borg version, produce a corresponding sequence of
-    command-line flags.
+    Given a retention config dict mapping from option name to value, tranform it into an iterable of
+    command-line name-value flag pairs.
 
     For example, given a retention config of:
 
@@ -25,85 +22,60 @@ def make_prune_flags(config, prune_arguments, local_borg_version):
             ('--keep-monthly', '6'),
         )
     '''
-    flag_pairs = (
-        ('--' + option_name.replace('_', '-'), str(value))
-        for option_name, value in config.items()
-        if option_name.startswith('keep_') and option_name != 'keep_exclude_tags'
-    )
-    prefix = config.get('prefix')
+    config = retention_config.copy()
 
-    return tuple(element for pair in flag_pairs for element in pair) + (
-        (
-            ('--match-archives', f'sh:{prefix}*')
-            if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
-            else ('--glob-archives', f'{prefix}*')
-        )