.flake8

@@ -0,0 +1 @@
+select = Q0

.gitea/workflows/build.yaml

@@ -26,5 +26,3 @@ jobs:
           PASSWORD: "${{ secrets.REGISTRY_PASSWORD }}"
       - run: podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
       - run: podman push "$IMAGE_NAME"
-      - run: scripts/export-docs-from-image
-      - run: curl --user "${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }}" --upload-file borgmatic-docs.tar.gz https://projects.torsion.org/api/packages/borgmatic-collective/generic/borgmatic-docs/$(head --lines=1 NEWS)/borgmatic-docs.tar.gz

NEWS

@@ -1,306 +1,3 @@
-2.0.0.dev0
- * #262: Add a "default_actions" option that supports disabling default actions when borgmatic is
-   run without any command-line arguments.
- * #345: Add a "key import" action to import a repository key from backup.
- * #422: Add home directory expansion to file-based and KeePassXC credential hooks.
- * #610: Add a "recreate" action for recreating archives, for instance for retroactively excluding
-   particular files from existing archives.
- * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more
-   flexible "commands:". See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
- * #790: BREAKING: For both new and deprecated command hooks, run a configured "after" hook even if
-   an error occurs first. This allows you to perform cleanup steps that correspond to "before"
-   preparation commands—even when something goes wrong.
- * #790: BREAKING: Run all command hooks (both new and deprecated) respecting the
-   "working_directory" option if configured, meaning that hook commands are run in that directory.
- * #836: Add a custom command option for the SQLite hook.
- * #837: Add custom command options for the MongoDB hook.
- * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune".
- * #1020: Document a database use case involving a temporary database client container:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
- * #1037: Fix an error with the "extract" action when both a remote repository and a
-   "working_directory" are used.
- * #1044: Fix an error in the systemd credential hook when the credential name contains a "."
-   character.
-
-1.9.14
- * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the
-   incident UI. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook
- * #936: Clarify Zabbix monitoring hook documentation about creating items:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook
- * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly.
- * #1021: Fix a regression in which the "exclude_patterns" option didn't expand "~" (the user's
-   home directory). This fix means that all "patterns" and "patterns_from" also now expand "~".
- * #1023: Fix an error in the Btrfs hook when attempting to snapshot a read-only subvolume. Now,
-   read-only subvolumes are ignored since Btrfs can't actually snapshot them.
-
-1.9.13
- * #975: Add a "compression" option to the PostgreSQL database hook.
- * #1001: Fix a ZFS error during snapshot cleanup.
- * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes.
- * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than
-   using an environment variable.
- * #1013: Send database passwords to MongoDB via anonymous pipe, which is more secure than using
-   "--password" on the command-line!
- * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit.
- * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification.
- * Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption
-   between client and server.
-
-1.9.12
- * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will
-   work with Python 3.9, 3.10, and 3.11 again.
-
-1.9.11
- * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
- * #996: Fix the "create" action to omit the repository label prefix from Borg's output when
-   databases are enabled.
- * #998: Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure
-   than using an environment variable.
- * #999: Fix a runtime directory error from a conflict between "extra_borg_options" and special file
-   detection.
- * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from
-   a borgmatic configuration option (e.g. "source_directories")—not from other hooks within
-   borgmatic.
- * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical
-   volumes.
- * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property.
- * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
-   source directories.
-
-1.9.10
- * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic
-   configuration files. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
- * #987: Fix a "list" action error when the "encryption_passcommand" option is set.
- * #987: When both "encryption_passcommand" and "encryption_passphrase" are configured, prefer
-   "encryption_passphrase" even if it's an empty value.
- * #988: With the "max_duration" option or the "--max-duration" flag, run the archives and
-   repository checks separately so they don't interfere with one another. Previously, borgmatic
-   refused to run checks in this situation.
- * #989: Fix the log message code to avoid using Python 3.10+ logging features. Now borgmatic will
-   work with Python 3.9 again.
- * Capture and delay any log records produced before logging is fully configured, so early log
-   records don't get lost.
- * Add support for Python 3.13.
-
-1.9.9
- * #635: Log the repository path or label on every relevant log message, not just some logs.
- * #961: When the "encryption_passcommand" option is set, call the command once from borgmatic to
-   collect the encryption passphrase and then pass it to Borg multiple times. See the documentation
-   for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
- * #981: Fix a "spot" check file count delta error.
- * #982: Fix for borgmatic "exclude_patterns" and "exclude_from" recursing into excluded
-   subdirectories.
- * #983: Fix the Btrfs hook to support subvolumes with names like "@home" different from their
-   mount points.
- * #985: Change the default value for the "--original-hostname" flag from "localhost" to no host
-   specified. This way, the "restore" action works without a hostname if there's a single matching
-   database dump.
-
-1.9.8
- * #979: Fix root patterns so they don't have an invalid "sh:" prefix before getting passed to Borg.
- * Expand the recent contributors documentation section to include ticket submitters—not just code
-   contributors—because there are multiple ways to contribute to the project! See:
-   https://torsion.org/borgmatic/#recent-contributors
-
-1.9.7
- * #855: Add a Sentry monitoring hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook
- * #968: Fix for a "spot" check error when a filename in the most recent archive contains a newline.
- * #970: Fix for an error when there's a blank line in the configured patterns or excludes.
- * #971: Fix for "exclude_from" files being completely ignored.
- * #977: Fix for "exclude_patterns" and "exclude_from" not supporting explicit pattern styles (e.g.,
-   "sh:" or "re:").
-
-1.9.6
- * #959: Fix an error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
-   source directories.
- * #960: Fix for archives storing relative source directory paths such that they contain the working
-   directory.
- * #960: Fix the "spot" check to support relative source directory paths.
- * #962: For the ZFS, Btrfs, and LVM hooks, perform path rewriting for excludes and patterns in
-   addition to the existing source directories rewriting.
- * #962: Under the hood, merge all configured source directories, excludes, and patterns into a
-   unified temporary patterns file for passing to Borg. The borgmatic configuration options remain
-   unchanged.
- * #962: For the LVM hook, add support for nested logical volumes.
- * #965: Fix a borgmatic runtime directory error when running the "spot" check with a database hook
-   enabled.
- * #969: Fix the "restore" action to work on database dumps without a port when a default port is
-   present in configuration.
- * Fix the "spot" check to no longer consider pipe files within an archive for file comparisons.
- * Fix the "spot" check to have a nicer error when there are no source paths to compare.
- * Fix auto-excluding of special files (when databases are configured) to support relative source
-   directory paths.
- * Drop support for Python 3.8, which has been end-of-lifed.
-
-1.9.5
- * #418: Backup and restore databases that have the same name but with different ports, hostnames,
-   or hooks.
- * #947: To avoid a hang in the database hooks, error and exit when the borgmatic runtime
-   directory overlaps with the configured excludes.
- * #954: Fix a findmnt command error in the Btrfs hook by switching to parsing JSON output.
- * #956: Fix the printing of a color reset code even when color is disabled.
- * #958: Drop colorama as a library dependency.
- * When the ZFS, Btrfs, or LVM hooks aren't configured, don't try to cleanup snapshots for them.
-
-1.9.4
- * #80 (beta): Add an LVM hook for snapshotting and backing up LVM logical volumes. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
- * #251 (beta): Add a Btrfs hook for snapshotting and backing up Btrfs subvolumes. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
- * #926: Fix a library error when running within a PyInstaller bundle.
- * #950: Fix a snapshot unmount error in the ZFS hook when using nested datasets.
- * Update the ZFS hook to discover and snapshot ZFS datasets even if they are parent/grandparent
-   directories of your source directories.
- * Reorganize data source and monitoring hooks to make developing new hooks easier.
-
-1.9.3
- * #261 (beta): Add a ZFS hook for snapshotting and backing up ZFS datasets. See the documentation
-   for more information: https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
- * Remove any temporary copies of the manifest file created in support of the "bootstrap" action.
- * Deprecate the "store_config_files" option at the global scope and move it under the "bootstrap"
-   hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive
- * Require the runtime directory to be an absolute path.
- * Add a "--deleted" flag to the "repo-list" action for listing deleted archives that haven't
-   yet been compacted (Borg 2 only).
- * Promote the "spot" check from a beta feature to stable.
-
-1.9.2
- * #441: Apply the "umask" option to all relevant actions, not just some of them.
- * #722: Remove the restriction that the "extract" and "mount" actions must match a single
-   repository. Now they work more like other actions, where each repository is applied in turn.
- * #932: Fix the missing build backend setting in pyproject.toml to allow Fedora builds.
- * #934: Update the logic that probes for the borgmatic streaming database dump, bootstrap
-   metadata, and check state directories to support more platforms and use cases. See the
-   documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory
- * #934: Add the "RuntimeDirectory" and "StateDirectory" options to the sample systemd service
-   file to support the new runtime and state directory logic.
- * #939: Fix borgmatic ignoring the "BORG_RELOCATED_REPO_ACCESS_IS_OK" and
-   "BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK" environment variables.
- * Add a Pushover monitoring hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook
-
-1.9.1
- * #928: Fix the user runtime directory location on macOS (and possibly Cygwin).
- * #930: Fix an error with the sample systemd service when no credentials are configured.
- * #931: Fix an error when implicitly upgrading the check state directory from ~/.borgmatic to
-   ~/.local/state/borgmatic across filesystems.
-
-1.9.0
- * #609: Fix the glob expansion of "source_directories" values to respect the "working_directory"
-   option.
- * #609: BREAKING: Apply the "working_directory" option to all actions, not just "create". This
-   includes repository paths, destination paths, mount points, etc.
- * #562: Deprecate the "borgmatic_source_directory" option in favor of "user_runtime_directory"
-   and "user_state_directory".
- * #562: BREAKING: Move the default borgmatic streaming database dump and bootstrap metadata
-   directory from ~/.borgmatic to /run/user/$UID/borgmatic, which is more XDG-compliant. You can
-   override this location with the new "user_runtime_directory" option. Existing archives with
-   database dumps at the old location are still restorable. 
- * #562, #638: Move the default check state directory from ~/.borgmatic to 
-   ~/.local/state/borgmatic. This is more XDG-compliant and also prevents these state files from
-   getting backed up (unless you explicitly include them). You can override this location with the
-   new "user_state_directory" option. After the first time you run the "check" action with borgmatic
-   1.9.0, you can safely delete the ~/.borgmatic directory.
- * #838: BREAKING: With Borg 1.4+, store database dumps and bootstrap metadata in a "/borgmatic"
-   directory within a backup archive, so the path doesn't depend on the current user. This means
-   that you can now backup as one user and restore or bootstrap as another user, among other use
-   cases.
- * #902: Add loading of encrypted systemd credentials. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/#using-systemd-service-credentials
- * #911: Add a "key change-passphrase" action to change the passphrase protecting a repository key.
- * #914: Fix a confusing apparent hang when when the repository location changes, and instead
-   show a helpful error message.
- * #915: BREAKING: Rename repository actions like "rcreate" to more explicit names like
-   "repo-create" for compatibility with recent changes in Borg 2.0.0b10.
- * #918: BREAKING: When databases are configured, don't auto-enable the "one_file_system" option,
-   as existing auto-excludes of special files should be sufficient to prevent Borg from hanging on
-   them. But if this change causes problems for you, you can always enable "one_file_system"
-   explicitly.
- * #919: Clarify the command-line help for the "--config" flag.
- * #919: Document a policy for versioning and breaking changes:
-   https://torsion.org/borgmatic/docs/how-to/upgrade/#versioning-and-breaking-changes
- * #921: BREAKING: Change soft failure command hooks to skip only the current repository rather than
-   all repositories in the configuration file.
- * #922: Replace setup.py (Python packaging metadata) with the more modern pyproject.toml.
- * When using Borg 2, default the "archive_name_format" option to just "{hostname}", as Borg 2 does
-   not require unique archive names; identical archive names form a common "series" that can be
-   targeted together. See the Borg 2 documentation for more information:
-   https://borgbackup.readthedocs.io/en/2.0.0b13/changes.html#borg-1-2-x-1-4-x-to-borg-2-0
- * Add support for Borg 2's "rclone:" repository URLs, so you can backup to 70+ cloud storage
-   services whether or not they support Borg explicitly.
- * Add support for Borg 2's "sftp://" repository URLs.
- * Update the "--match-archives" and "--archive" flags to support Borg 2 series names or archive
-   hashes.
- * Add a "--match-archives" flag to the "prune" action.
- * Add "--local-path" and "--remote-path" flags to the "config bootstrap" action for setting the
-   Borg executable paths used for bootstrapping.
- * Add a "--user-runtime-directory" flag to the "config bootstrap" action for helping borgmatic
-   locate the bootstrap metadata stored in an archive.
- * Add a Zabbix monitoring hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook
- * Add a tarball of borgmatic's HTML documentation to the packages on the project page.
-
-1.8.14
- * #896: Fix an error in borgmatic rcreate/init on an empty repository directory with Borg 1.4.
- * #898: Add glob ("*") support to the "--repository" flag. Just quote any values containing
-   globs so your shell doesn't interpret them.
- * #899: Fix for a "bad character" Borg error in which the "spot" check fed Borg an invalid pattern.
- * #900: Fix for a potential traceback (TypeError) during the handling of another error.
- * #904: Clarify the configuration reference about the "spot" check options:
-   https://torsion.org/borgmatic/docs/reference/configuration/
- * #905: Fix the "source_directories_must_exist" option to work with relative "source_directories"
-   paths when a "working_directory" is set.
- * #906: Add documentation details for how to run custom database dump commands using binaries from
-   running containers:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
- * Fix a regression in which the "color" option had no effect.
- * Add a recent contributors section to the documentation, because credit where credit's due! See:
-   https://torsion.org/borgmatic/#recent-contributors
-
-1.8.13
- * #298: Add "delete" and "rdelete" actions to delete archives or entire repositories.
- * #785: Add an "only_run_on" option to consistency checks so you can limit a check to running on
-   particular days of the week. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#check-days
- * #885: Add an Uptime Kuma monitoring hook. See the documentation for more information:
-   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook
- * #886: Fix a PagerDuty hook traceback with Python < 3.10.
- * #889: Fix the Healthchecks ping body size limit, restoring it to the documented 100,000 bytes.
-
-1.8.12
- * #817: Add a "--max-duration" flag to the "check" action and a "max_duration" option to the
-   repository check configuration. This tells Borg to interrupt a repository check after a certain
-   duration.
- * #860: Fix interaction between environment variable interpolation in constants and shell escaping.
- * #863: When color output is disabled (explicitly or implicitly), don't prefix each log line with
-   the log level.
- * #865: Add an "upload_buffer_size" option to set the size of the upload buffer used in "create"
-   action.
- * #866: Fix "Argument list too long" error in the "spot" check when checking hundreds of thousands
-   of files at once.
- * #874: Add the configured repository label as "repository_label" to the interpolated variables
-   passed to before/after command hooks.
- * #881: Fix "Unrecognized argument" error when the same value is used with different command-line
-   flags.
- * In the "spot" check, don't try to hash symlinked directories.
-
-1.8.11
- * #815: Add optional Healthchecks auto-provisioning via "create_slug" option.
- * #851: Fix lack of file extraction when using "extract --strip-components all" on a path with a
-   leading slash.
- * #854: Fix a traceback when the "data" consistency check is used.
- * #857: Fix a traceback with "check --only spot" when the "spot" check is unconfigured.
-
 1.8.10
  * #656 (beta): Add a "spot" consistency check that compares file counts and contents between your
    source files and the latest archive, ensuring they fall within configured tolerances. This can

README.md

@@ -56,41 +56,19 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 
 ## Integrations
 
-### Data
-
 <a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://openzfs.org/"><img src="docs/static/openzfs.png" alt="OpenZFS" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://btrfs.readthedocs.io/"><img src="docs/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sourceware.org/lvm2/"><img src="docs/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://rclone.org"><img src="docs/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-
-
-### Monitoring
-
 <a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://uptime.kuma.pet/"><img src="docs/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.pushover.net/"><img src="docs/static/pushover.png" alt="Pushover" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://grafana.com/oss/loki/"><img src="docs/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.zabbix.com/"><img src="docs/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sentry.io/"><img src="docs/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-
-
-### Credentials
-
-<a href="https://systemd.io/"><img src="docs/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.docker.com/"><img src="docs/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://podman.io/"><img src="docs/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://keepassxc.org/"><img src="docs/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 
 ## Getting started
@@ -176,11 +154,3 @@ general, contributions are very welcome. We don't bite!
 Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
-
-### Recent contributors
-
-Thanks to all borgmatic contributors! There are multiple ways to contribute to
-this project, so the following includes those who have fixed bugs, contributed
-features, *or* filed tickets.
-
-{% include borgmatic/contributors.html %}

borgmatic/actions/borg.py

@@ -1,7 +1,7 @@
 import logging
 
 import borgmatic.borg.borg
-import borgmatic.borg.repo_list
+import borgmatic.borg.rlist
 import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
@@ -22,8 +22,10 @@ def run_borg(
     if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, borg_arguments.repository
     ):
-        logger.info('Running arbitrary Borg command')
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Running arbitrary Borg command'
+        )
+        archive_name = borgmatic.borg.rlist.resolve_archive_name(
             repository['path'],
             borg_arguments.archive,
             config,

borgmatic/actions/break_lock.py

@@ -21,7 +21,9 @@ def run_break_lock(
     if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, break_lock_arguments.repository
     ):
-        logger.info('Breaking repository and cache locks')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Breaking repository and cache locks'
+        )
         borgmatic.borg.break_lock.break_lock(
             repository['path'],
             config,

borgmatic/actions/change_passphrase.py

@@ -1,36 +0,0 @@
-import logging
-
-import borgmatic.borg.change_passphrase
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_change_passphrase(
-    repository,
-    config,
-    local_borg_version,
-    change_passphrase_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "key change-passphrase" action for the given repository.
-    '''
-    if (
-        change_passphrase_arguments.repository is None
-        or borgmatic.config.validate.repositories_match(
-            repository, change_passphrase_arguments.repository
-        )
-    ):
-        logger.info('Changing repository passphrase')
-        borgmatic.borg.change_passphrase.change_passphrase(
-            repository['path'],
-            config,
-            local_borg_version,
-            change_passphrase_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/check.py

@@ -1,4 +1,3 @@
-import calendar
 import datetime
 import hashlib
 import itertools
@@ -6,17 +5,14 @@ import logging
 import os
 import pathlib
 import random
-import shutil
 
-import borgmatic.actions.create
 import borgmatic.borg.check
 import borgmatic.borg.create
 import borgmatic.borg.environment
 import borgmatic.borg.extract
 import borgmatic.borg.list
-import borgmatic.borg.repo_list
+import borgmatic.borg.rlist
 import borgmatic.borg.state
-import borgmatic.config.paths
 import borgmatic.config.validate
 import borgmatic.execute
 import borgmatic.hooks.command
@@ -103,17 +99,12 @@ def parse_frequency(frequency):
         raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
 
 
-WEEKDAY_DAYS = calendar.day_name[0:5]
-WEEKEND_DAYS = calendar.day_name[5:7]
-
-
 def filter_checks_on_frequency(
     config,
     borg_repository_id,
     checks,
     force,
     archives_check_id=None,
-    datetime_now=datetime.datetime.now,
 ):
     '''
     Given a configuration dict with a "checks" sequence of dicts, a Borg repository ID, a sequence
@@ -152,29 +143,6 @@ def filter_checks_on_frequency(
         if checks and check not in checks:
             continue
 
-        only_run_on = check_config.get('only_run_on')
-        if only_run_on:
-            # Use a dict instead of a set to preserve ordering.
-            days = dict.fromkeys(only_run_on)
-
-            if 'weekday' in days:
-                days = {
-                    **dict.fromkeys(day for day in days if day != 'weekday'),
-                    **dict.fromkeys(WEEKDAY_DAYS),
-                }
-            if 'weekend' in days:
-                days = {
-                    **dict.fromkeys(day for day in days if day != 'weekend'),
-                    **dict.fromkeys(WEEKEND_DAYS),
-                }
-
-            if calendar.day_name[datetime_now().weekday()] not in days:
-                logger.info(
-                    f"Skipping {check} check due to day of the week; check only runs on {'/'.join(days)} (use --force to check anyway)"
-                )
-                filtered_checks.remove(check)
-                continue
-
         frequency_delta = parse_frequency(check_config.get('frequency'))
         if not frequency_delta:
             continue
@@ -185,8 +153,8 @@ def filter_checks_on_frequency(
 
         # If we've not yet reached the time when the frequency dictates we're ready for another
         # check, skip this check.
-        if datetime_now() < check_time + frequency_delta:
-            remaining = check_time + frequency_delta - datetime_now()
+        if datetime.datetime.now() < check_time + frequency_delta:
+            remaining = check_time + frequency_delta - datetime.datetime.now()
             logger.info(
                 f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
             )
@@ -212,11 +180,15 @@ def make_check_time_path(config, borg_repository_id, check_type, archives_check_
     "archives", etc.), and a unique hash of the archives filter flags, return a path for recording
     that check's time (the time of that check last occurring).
     '''
-    borgmatic_state_directory = borgmatic.config.paths.get_borgmatic_state_directory(config)
+    borgmatic_source_directory = os.path.expanduser(
+        config.get(
+            'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+        )
+    )
 
     if check_type in ('archives', 'data'):
         return os.path.join(
-            borgmatic_state_directory,
+            borgmatic_source_directory,
             'checks',
             borg_repository_id,
             check_type,
@@ -224,7 +196,7 @@ def make_check_time_path(config, borg_repository_id, check_type, archives_check_
         )
 
     return os.path.join(
-        borgmatic_state_directory,
+        borgmatic_source_directory,
         'checks',
         borg_repository_id,
         check_type,
@@ -257,7 +229,7 @@ def read_check_time(path):
 def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
     '''
     Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
-    "archives", etc.), and a unique hash of the archives filter flags, return the corresponding
+    "archives", etc.), and a unique hash of the archives filter flags, return a the corresponding
     check time or None if such a check time does not exist.
 
     When the check type is "archives" or "data", this function probes two different paths to find
@@ -295,37 +267,14 @@ def upgrade_check_times(config, borg_repository_id):
     Given a configuration dict and a Borg repository ID, upgrade any corresponding check times on
     disk from old-style paths to new-style paths.
 
-    One upgrade performed is moving the checks directory from:
+    Currently, the only upgrade performed is renaming an archive or data check path that looks like:
 
-      {borgmatic_source_directory}/checks (e.g., ~/.borgmatic/checks)
+      ~/.borgmatic/checks/1234567890/archives
 
     to:
 
-      {borgmatic_state_directory}/checks (e.g. ~/.local/state/borgmatic)
-
-    Another upgrade is renaming an archive or data check path that looks like:
-
-      {borgmatic_state_directory}/checks/1234567890/archives
-
-    to:
-
-      {borgmatic_state_directory}/checks/1234567890/archives/all
+      ~/.borgmatic/checks/1234567890/archives/all
     '''
-    borgmatic_source_checks_path = os.path.join(
-        borgmatic.config.paths.get_borgmatic_source_directory(config), 'checks'
-    )
-    borgmatic_state_path = borgmatic.config.paths.get_borgmatic_state_directory(config)
-    borgmatic_state_checks_path = os.path.join(borgmatic_state_path, 'checks')
-
-    if os.path.exists(borgmatic_source_checks_path) and not os.path.exists(
-        borgmatic_state_checks_path
-    ):
-        logger.debug(
-            f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}'
-        )
-        os.makedirs(borgmatic_state_path, mode=0o700, exist_ok=True)
-        shutil.move(borgmatic_source_checks_path, borgmatic_state_checks_path)
-
     for check_type in ('archives', 'data'):
         new_path = make_check_time_path(config, borg_repository_id, check_type, 'all')
         old_path = os.path.dirname(new_path)
@@ -334,104 +283,92 @@ def upgrade_check_times(config, borg_repository_id):
         if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
             continue
 
-        logger.debug(f'Upgrading archives check time file from {old_path} to {new_path}')
+        logger.debug(f'Upgrading archives check time from {old_path} to {new_path}')
 
         try:
-            shutil.move(old_path, temporary_path)
+            os.rename(old_path, temporary_path)
         except FileNotFoundError:
             pass
 
         os.mkdir(old_path)
-        shutil.move(temporary_path, new_path)
+        os.rename(temporary_path, new_path)
 
 
 def collect_spot_check_source_paths(
-    repository,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path,
-    remote_path,
-    borgmatic_runtime_directory,
+    repository, config, local_borg_version, global_arguments, local_path, remote_path
 ):
     '''
     Given a repository configuration dict, a configuration dict, the local Borg version, global
     arguments as an argparse.Namespace instance, the local Borg path, and the remote Borg path,
-    collect the source paths that Borg would use in an actual create (but only include files).
+    collect the source paths that Borg would use in an actual create (but only include files and
+    symlinks).
     '''
     stream_processes = any(
         borgmatic.hooks.dispatch.call_hooks(
             'use_streaming',
             config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+            repository['path'],
+            borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
         ).values()
     )
-    working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    (create_flags, create_positional_arguments, pattern_file) = (
+    (create_flags, create_positional_arguments, pattern_file, exclude_file) = (
         borgmatic.borg.create.make_base_create_command(
             dry_run=True,
             repository_path=repository['path'],
             config=config,
-            patterns=borgmatic.actions.create.process_patterns(
-                borgmatic.actions.create.collect_patterns(config),
-                working_directory,
-            ),
+            config_paths=(),
             local_borg_version=local_borg_version,
             global_arguments=global_arguments,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
+            borgmatic_source_directories=(),
             local_path=local_path,
             remote_path=remote_path,
             list_files=True,
             stream_processes=stream_processes,
         )
     )
-    working_directory = borgmatic.config.paths.get_working_directory(config)
+    borg_environment = borgmatic.borg.environment.make_environment(config)
+
+    try:
+        working_directory = os.path.expanduser(config.get('working_directory'))
+    except TypeError:
+        working_directory = None
 
     paths_output = borgmatic.execute.execute_command_and_capture_output(
         create_flags + create_positional_arguments,
         capture_stderr=True,
-        environment=borgmatic.borg.environment.make_environment(config),
         working_directory=working_directory,
+        extra_environment=borg_environment,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
 
     paths = tuple(
         path_line.split(' ', 1)[1]
-        for path_line in paths_output.splitlines()
+        for path_line in paths_output.split('\n')
         if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
     )
 
-    return tuple(
-        path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
-    )
+    return tuple(path for path in paths if os.path.isfile(path) or os.path.islink(path))
 
 
 BORG_DIRECTORY_FILE_TYPE = 'd'
-BORG_PIPE_FILE_TYPE = 'p'
 
 
 def collect_spot_check_archive_paths(
-    repository,
-    archive,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path,
-    remote_path,
-    borgmatic_runtime_directory,
+    repository, archive, config, local_borg_version, global_arguments, local_path, remote_path
 ):
     '''
     Given a repository configuration dict, the name of the latest archive, a configuration dict, the
-    local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the
-    remote Borg path, and the borgmatic runtime directory, collect the paths from the given archive
-    (but only include files and symlinks and exclude borgmatic runtime directories).
-
-    These paths do not have a leading slash, as that's how Borg stores them. As a result, we don't
-    know whether they came from absolute or relative source directories.
+    local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, and
+    the remote Borg path, collect the paths from the given archive (but only include files and
+    symlinks).
     '''
-    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
+    borgmatic_source_directory = os.path.expanduser(
+        config.get(
+            'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+        )
+    )
 
     return tuple(
         path
@@ -441,23 +378,16 @@ def collect_spot_check_archive_paths(
             config,
             local_borg_version,
             global_arguments,
-            path_format='{type} {path}{NUL}',  # noqa: FS003
+            path_format='{type} /{path}{NL}',  # noqa: FS003
             local_path=local_path,
             remote_path=remote_path,
         )
         for (file_type, path) in (line.split(' ', 1),)
-        if file_type not in (BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE)
-        if pathlib.Path('borgmatic') not in pathlib.Path(path).parents
-        if pathlib.Path(borgmatic_source_directory.lstrip(os.path.sep))
-        not in pathlib.Path(path).parents
-        if pathlib.Path(borgmatic_runtime_directory.lstrip(os.path.sep))
-        not in pathlib.Path(path).parents
+        if file_type != BORG_DIRECTORY_FILE_TYPE
+        if pathlib.Path(borgmatic_source_directory) not in pathlib.Path(path).parents
     )
 
 
-SAMPLE_PATHS_SUBSET_COUNT = 10000
-
-
 def compare_spot_check_hashes(
     repository,
     archive,
@@ -466,14 +396,15 @@ def compare_spot_check_hashes(
     global_arguments,
     local_path,
     remote_path,
+    log_label,
     source_paths,
 ):
     '''
     Given a repository configuration dict, the name of the latest archive, a configuration dict, the
     local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the
-    remote Borg path, and spot check source paths, compare the hashes for a sampling of the source
-    paths with hashes from corresponding paths in the given archive. Return a sequence of the paths
-    that fail that hash comparison.
+    remote Borg path, a log label, and spot check source paths, compare the hashes for a sampling of
+    the source paths with hashes from corresponding paths in the given archive. Return a sequence of
+    the paths that fail that hash comparison.
     '''
     # Based on the configured sample percentage, come up with a list of random sample files from the
     # source directories.
@@ -482,74 +413,45 @@ def compare_spot_check_hashes(
         int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
     )
     source_sample_paths = tuple(random.sample(source_paths, sample_count))
-    working_directory = borgmatic.config.paths.get_working_directory(config)
     existing_source_sample_paths = {
-        source_path
-        for source_path in source_sample_paths
-        if os.path.exists(os.path.join(working_directory or '', source_path))
+        source_path for source_path in source_sample_paths if os.path.exists(source_path)
     }
     logger.debug(
-        f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
+        f'{log_label}: Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
     )
 
-    source_sample_paths_iterator = iter(source_sample_paths)
-    source_hashes = {}
-    archive_hashes = {}
+    # Hash each file in the sample paths (if it exists).
+    hash_output = borgmatic.execute.execute_command_and_capture_output(
+        (spot_check_config.get('xxh64sum_command', 'xxh64sum'),)
+        + tuple(path for path in source_sample_paths if path in existing_source_sample_paths)
+    )
 
-    # Only hash a few thousand files at a time (a subset of the total paths) to avoid an "Argument
-    # list too long" shell error.
-    while True:
-        # Hash each file in the sample paths (if it exists).
-        source_sample_paths_subset = tuple(
-            itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT)
-        )
-        if not source_sample_paths_subset:
-            break
+    source_hashes = dict(
+        (reversed(line.split('  ', 1)) for line in hash_output.splitlines()),
+        **{path: '' for path in source_sample_paths if path not in existing_source_sample_paths},
+    )
 
-        hash_output = borgmatic.execute.execute_command_and_capture_output(
-            (spot_check_config.get('xxh64sum_command', 'xxh64sum'),)
-            + tuple(
-                path for path in source_sample_paths_subset if path in existing_source_sample_paths
-            ),
-            working_directory=working_directory,
-        )
-
-        source_hashes.update(
-            **dict(
-                (reversed(line.split('  ', 1)) for line in hash_output.splitlines()),
-                # Represent non-existent files as having empty hashes so the comparison below still works.
-                **{
-                    path: ''
-                    for path in source_sample_paths_subset
-                    if path not in existing_source_sample_paths
-                },
-            )
-        )
-
-        # Get the hash for each file in the archive.
-        archive_hashes.update(
-            **dict(
-                reversed(line.split(' ', 1))
-                for line in borgmatic.borg.list.capture_archive_listing(
-                    repository['path'],
-                    archive,
-                    config,
-                    local_borg_version,
-                    global_arguments,
-                    list_paths=source_sample_paths_subset,
-                    path_format='{xxh64} {path}{NUL}',  # noqa: FS003
-                    local_path=local_path,
-                    remote_path=remote_path,
-                )
-                if line
-            )
+    archive_hashes = dict(
+        reversed(line.split(' ', 1))
+        for line in borgmatic.borg.list.capture_archive_listing(
+            repository['path'],
+            archive,
+            config,
+            local_borg_version,
+            global_arguments,
+            list_paths=source_sample_paths,
+            path_format='{xxh64} /{path}{NL}',  # noqa: FS003
+            local_path=local_path,
+            remote_path=remote_path,
         )
+        if line
+    )
 
     # Compare the source hashes with the archive hashes to see how many match.
     failing_paths = []
 
     for path, source_hash in source_hashes.items():
-        archive_hash = archive_hashes.get(path.lstrip(os.path.sep))
+        archive_hash = archive_hashes.get(path)
 
         if archive_hash is not None and archive_hash == source_hash:
             continue
@@ -566,25 +468,19 @@ def spot_check(
     global_arguments,
     local_path,
     remote_path,
-    borgmatic_runtime_directory,
 ):
     '''
     Given a repository dict, a loaded configuration dict, the local Borg version, global arguments
-    as an argparse.Namespace instance, the local Borg path, the remote Borg path, and the borgmatic
-    runtime directory, perform a spot check for the latest archive in the given repository.
+    as an argparse.Namespace instance, the local Borg path, and the remote Borg path, perform a spot
+    check for the latest archive in the given repository.
 
     A spot check compares file counts and also the hashes for a random sampling of source files on
     disk to those stored in the latest archive. If any differences are beyond configured tolerances,
     then the check fails.
     '''
-    logger.debug('Running spot check')
-
-    try:
-        spot_check_config = next(
-            check for check in config.get('checks', ()) if check.get('name') == 'spot'
-        )
-    except StopIteration:
-        raise ValueError('Cannot run spot check because it is unconfigured')
+    log_label = f'{repository.get("label", repository["path"])}'
+    logger.debug(f'{log_label}: Running spot check')
+    spot_check_config = next(check for check in config['checks'] if check['name'] == 'spot')
 
     if spot_check_config['data_tolerance_percentage'] > spot_check_config['data_sample_percentage']:
         raise ValueError(
@@ -598,11 +494,10 @@ def spot_check(
         global_arguments,
         local_path,
         remote_path,
-        borgmatic_runtime_directory,
     )
-    logger.debug(f'{len(source_paths)} total source paths for spot check')
+    logger.debug(f'{log_label}: {len(source_paths)} total source paths for spot check')
 
-    archive = borgmatic.borg.repo_list.resolve_archive_name(
+    archive = borgmatic.borg.rlist.resolve_archive_name(
         repository['path'],
         'latest',
         config,
@@ -611,7 +506,7 @@ def spot_check(
         local_path,
         remote_path,
     )
-    logger.debug(f'Using archive {archive} for spot check')
+    logger.debug(f'{log_label}: Using archive {archive} for spot check')
 
     archive_paths = collect_spot_check_archive_paths(
         repository,
@@ -621,29 +516,19 @@ def spot_check(
         global_arguments,
         local_path,
         remote_path,
-        borgmatic_runtime_directory,
     )
-    logger.debug(f'{len(archive_paths)} total archive paths for spot check')
-
-    if len(source_paths) == 0:
-        logger.debug(
-            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths)) or "none"}'
-        )
-        raise ValueError(
-            'Spot check failed: There are no source paths to compare against the archive'
-        )
+    logger.debug(f'{log_label}: {len(archive_paths)} total archive paths for spot check')
 
     # Calculate the percentage delta between the source paths count and the archive paths count, and
     # compare that delta to the configured count tolerance percentage.
     count_delta_percentage = abs(len(source_paths) - len(archive_paths)) / len(source_paths) * 100
 
     if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
-        rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths)
         logger.debug(
-            f'Paths in source paths but not latest archive: {", ".join(rootless_source_paths - set(archive_paths)) or "none"}'
+            f'{log_label}: Paths in source paths but not latest archive: {", ".join(set(source_paths) - set(archive_paths)) or "none"}'
         )
         logger.debug(
-            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths) - rootless_source_paths) or "none"}'
+            f'{log_label}: Paths in latest archive but not source paths: {", ".join(set(archive_paths) - set(source_paths)) or "none"}'
         )
         raise ValueError(
             f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'
@@ -657,24 +542,25 @@ def spot_check(
         global_arguments,
         local_path,
         remote_path,
+        log_label,
         source_paths,
     )
 
     # Error if the percentage of failing hashes exceeds the configured tolerance percentage.
-    logger.debug(f'{len(failing_paths)} non-matching spot check hashes')
+    logger.debug(f'{log_label}: {len(failing_paths)} non-matching spot check hashes')
     data_tolerance_percentage = spot_check_config['data_tolerance_percentage']
     failing_percentage = (len(failing_paths) / len(source_paths)) * 100
 
     if failing_percentage > data_tolerance_percentage:
         logger.debug(
-            f'Source paths with data not matching the latest archive: {", ".join(failing_paths)}'
+            f'{log_label}: Source paths with data not matching the latest archive: {", ".join(failing_paths)}'
         )
         raise ValueError(
             f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)'
         )
 
     logger.info(
-        f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta'
+        f'{log_label}: Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta'
     )
 
 
@@ -682,6 +568,7 @@ def run_check(
     config_filename,
     repository,
     config,
+    hook_context,
     local_borg_version,
     check_arguments,
     global_arguments,
@@ -698,8 +585,16 @@ def run_check(
     ):
         return
 
-    logger.info('Running consistency checks')
+    borgmatic.hooks.command.execute_hook(
+        config.get('before_check'),
+        config.get('umask'),
+        config_filename,
+        'pre-check',
+        global_arguments.dry_run,
+        **hook_context,
+    )
 
+    logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks')
     repository_id = borgmatic.borg.check.get_repository_id(
         repository['path'],
         config,
@@ -751,14 +646,21 @@ def run_check(
         write_check_time(make_check_time_path(config, repository_id, 'extract'))
 
     if 'spot' in checks:
-        with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-            spot_check(
-                repository,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-                borgmatic_runtime_directory,
-            )
+        spot_check(
+            repository,
+            config,
+            local_borg_version,
+            global_arguments,
+            local_path,
+            remote_path,
+        )
         write_check_time(make_check_time_path(config, repository_id, 'spot'))
+
+    borgmatic.hooks.command.execute_hook(
+        config.get('after_check'),
+        config.get('umask'),
+        config_filename,
+        'post-check',
+        global_arguments.dry_run,
+        **hook_context,
+    )

borgmatic/actions/compact.py

@@ -12,6 +12,7 @@ def run_compact(
     config_filename,
     repository,
     config,
+    hook_context,
     local_borg_version,
     compact_arguments,
     global_arguments,
@@ -27,8 +28,18 @@ def run_compact(
     ):
         return
 
+    borgmatic.hooks.command.execute_hook(
+        config.get('before_compact'),
+        config.get('umask'),
+        config_filename,
+        'pre-compact',
+        global_arguments.dry_run,
+        **hook_context,
+    )
     if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
-        logger.info(f'Compacting segments{dry_run_label}')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}'
+        )
         borgmatic.borg.compact.compact_segments(
             global_arguments.dry_run,
             repository['path'],
@@ -42,4 +53,14 @@ def run_compact(
             threshold=compact_arguments.threshold,
         )
     else:  # pragma: nocover
-        logger.info('Skipping compact (only available/needed in Borg 1.2+)')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)'
+        )
+    borgmatic.hooks.command.execute_hook(
+        config.get('after_compact'),
+        config.get('umask'),
+        config_filename,
+        'post-compact',
+        global_arguments.dry_run,
+        **hook_context,
+    )

borgmatic/actions/config/bootstrap.py

@@ -3,78 +3,55 @@ import logging
 import os
 
 import borgmatic.borg.extract
-import borgmatic.borg.repo_list
-import borgmatic.config.paths
+import borgmatic.borg.rlist
 import borgmatic.config.validate
 import borgmatic.hooks.command
+from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
 
 logger = logging.getLogger(__name__)
 
 
-def make_bootstrap_config(bootstrap_arguments):
+def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
     '''
-    Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict.
-    '''
-    return {
-        'ssh_command': bootstrap_arguments.ssh_command,
-        # In case the repo has been moved or is accessed from a different path at the point of
-        # bootstrapping.
-        'relocated_repo_access_is_ok': True,
-    }
-
-
-def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version):
-    '''
-    Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the
-    repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory,
-    borgmatic source directory, destination directory, and whether to strip components), the global
-    arguments as an argparse.Namespace (containing the dry run flag and the local borg version),
-    return the config paths from the manifest.json file in the borgmatic source directory or runtime
-    directory after extracting it from the repository archive.
+    Given the bootstrap arguments as an argparse.Namespace (containing the repository and archive
+    name, borgmatic source directory, destination directory, and whether to strip components), the
+    global arguments as an argparse.Namespace (containing the dry run flag and the local borg
+    version), return the config paths from the manifest.json file in the borgmatic source directory
+    after extracting it from the repository.
 
     Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
     expected configuration path data.
     '''
-    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(
-        {'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory}
+    borgmatic_source_directory = (
+        bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY
     )
-    config = make_bootstrap_config(bootstrap_arguments)
+    borgmatic_manifest_path = os.path.expanduser(
+        os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
+    )
+    config = {'ssh_command': bootstrap_arguments.ssh_command}
 
-    # Probe for the manifest file in multiple locations, as the default location has moved to the
-    # borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But we
-    # still want to support reading the manifest from previously created archives as well.
-    with borgmatic.config.paths.Runtime_directory(
-        {'user_runtime_directory': bootstrap_arguments.user_runtime_directory},
-    ) as borgmatic_runtime_directory:
-        for base_directory in (
-            'borgmatic',
-            borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
-            borgmatic_source_directory,
-        ):
-            borgmatic_manifest_path = 'sh:' + os.path.join(
-                base_directory, 'bootstrap', 'manifest.json'
-            )
+    extract_process = borgmatic.borg.extract.extract_archive(
+        global_arguments.dry_run,
+        bootstrap_arguments.repository,
+        borgmatic.borg.rlist.resolve_archive_name(
+            bootstrap_arguments.repository,
+            bootstrap_arguments.archive,
+            config,
+            local_borg_version,
+            global_arguments,
+        ),
+        [borgmatic_manifest_path],
+        config,
+        local_borg_version,
+        global_arguments,
+        extract_to_stdout=True,
+    )
+    manifest_json = extract_process.stdout.read()
 
-            extract_process = borgmatic.borg.extract.extract_archive(
-                global_arguments.dry_run,
-                bootstrap_arguments.repository,
-                archive_name,
-                [borgmatic_manifest_path],
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path=bootstrap_arguments.local_path,
-                remote_path=bootstrap_arguments.remote_path,
-                extract_to_stdout=True,
-            )
-            manifest_json = extract_process.stdout.read()
-
-            if manifest_json:
-                break
-        else:
-            raise ValueError(
-                'Cannot read configuration paths from archive due to missing bootstrap manifest'
-            )
+    if not manifest_json:
+        raise ValueError(
+            'Cannot read configuration paths from archive due to missing bootstrap manifest'
+        )
 
     try:
         manifest_data = json.loads(manifest_json)
@@ -98,32 +75,27 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
     Raise ValueError if the bootstrap configuration could not be loaded.
     Raise CalledProcessError or OSError if Borg could not be run.
     '''
-    config = make_bootstrap_config(bootstrap_arguments)
-    archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-        bootstrap_arguments.repository,
-        bootstrap_arguments.archive,
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path=bootstrap_arguments.local_path,
-        remote_path=bootstrap_arguments.remote_path,
-    )
     manifest_config_paths = get_config_paths(
-        archive_name, bootstrap_arguments, global_arguments, local_borg_version
+        bootstrap_arguments, global_arguments, local_borg_version
     )
+    config = {'ssh_command': bootstrap_arguments.ssh_command}
 
     logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
 
     borgmatic.borg.extract.extract_archive(
         global_arguments.dry_run,
         bootstrap_arguments.repository,
-        archive_name,
+        borgmatic.borg.rlist.resolve_archive_name(
+            bootstrap_arguments.repository,
+            bootstrap_arguments.archive,
+            config,
+            local_borg_version,
+            global_arguments,
+        ),
         [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
         config,
         local_borg_version,
         global_arguments,
-        local_path=bootstrap_arguments.local_path,
-        remote_path=bootstrap_arguments.remote_path,
         extract_to_stdout=False,
         destination_path=bootstrap_arguments.destination,
         strip_components=bootstrap_arguments.strip_components,

borgmatic/actions/create.py

@@ -1,270 +1,46 @@
-import glob
-import itertools
+import importlib.metadata
+import json
 import logging
 import os
-import pathlib
 
 import borgmatic.actions.json
 import borgmatic.borg.create
-import borgmatic.borg.pattern
-import borgmatic.config.paths
+import borgmatic.borg.state
 import borgmatic.config.validate
 import borgmatic.hooks.command
 import borgmatic.hooks.dispatch
+import borgmatic.hooks.dump
 
 logger = logging.getLogger(__name__)
 
 
-def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_style.NONE):
+def create_borgmatic_manifest(config, config_paths, dry_run):
     '''
-    Given a Borg pattern as a string, parse it into a borgmatic.borg.pattern.Pattern instance and
-    return it.
+    Create a borgmatic manifest file to store the paths to the configuration files used to create
+    the archive.
     '''
-    try:
-        (pattern_type, remainder) = pattern_line.split(' ', maxsplit=1)
-    except ValueError:
-        raise ValueError(f'Invalid pattern: {pattern_line}')
+    if dry_run:
+        return
 
-    try:
-        (parsed_pattern_style, path) = remainder.split(':', maxsplit=1)
-        pattern_style = borgmatic.borg.pattern.Pattern_style(parsed_pattern_style)
-    except ValueError:
-        pattern_style = default_style
-        path = remainder
-
-    return borgmatic.borg.pattern.Pattern(
-        path,
-        borgmatic.borg.pattern.Pattern_type(pattern_type),
-        borgmatic.borg.pattern.Pattern_style(pattern_style),
-        source=borgmatic.borg.pattern.Pattern_source.CONFIG,
+    borgmatic_source_directory = config.get(
+        'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
     )
 
-
-def collect_patterns(config):
-    '''
-    Given a configuration dict, produce a single sequence of patterns comprised of the configured
-    source directories, patterns, excludes, pattern files, and exclude files.
-
-    The idea is that Borg has all these different ways of specifying includes, excludes, source
-    directories, etc., but we'd like to collapse them all down to one common format (patterns) for
-    ease of manipulation within borgmatic.
-    '''
-    try:
-        return (
-            tuple(
-                borgmatic.borg.pattern.Pattern(
-                    source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
-                )
-                for source_directory in config.get('source_directories', ())
-            )
-            + tuple(
-                parse_pattern(pattern_line.strip())
-                for pattern_line in config.get('patterns', ())
-                if not pattern_line.lstrip().startswith('#')
-                if pattern_line.strip()
-            )
-            + tuple(
-                parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
-                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                )
-                for exclude_line in config.get('exclude_patterns', ())
-            )
-            + tuple(
-                parse_pattern(pattern_line.strip())
-                for filename in config.get('patterns_from', ())
-                for pattern_line in open(filename).readlines()
-                if not pattern_line.lstrip().startswith('#')
-                if pattern_line.strip()
-            )
-            + tuple(
-                parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
-                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                )
-                for filename in config.get('exclude_from', ())
-                for exclude_line in open(filename).readlines()
-                if not exclude_line.lstrip().startswith('#')
-                if exclude_line.strip()
-            )
-        )
-    except (FileNotFoundError, OSError) as error:
-        logger.debug(error)
-
-        raise ValueError(f'Cannot read patterns_from/exclude_from file: {error.filename}')
-
-
-def expand_directory(directory, working_directory):
-    '''
-    Given a directory path, expand any tilde (representing a user's home directory) and any globs
-    therein. Return a list of one or more resulting paths.
-
-    Take into account the given working directory so that relative paths are supported.
-    '''
-    expanded_directory = os.path.expanduser(directory)
-
-    # This would be a lot easier to do with glob(..., root_dir=working_directory), but root_dir is
-    # only available in Python 3.10+.
-    normalized_directory = os.path.join(working_directory or '', expanded_directory)
-    glob_paths = glob.glob(normalized_directory)
-
-    if not glob_paths:
-        return [expanded_directory]
-
-    working_directory_prefix = os.path.join(working_directory or '', '')
-
-    return [
-        (
-            glob_path
-            # If these are equal, that means we didn't add any working directory prefix above.
-            if normalized_directory == expanded_directory
-            # Remove the working directory prefix that we added above in order to make glob() work.
-            # We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
-            # hack.
-            else glob_path.removeprefix(working_directory_prefix)
-        )
-        for glob_path in glob_paths
-    ]
-
-
-def expand_patterns(patterns, working_directory=None, skip_paths=None):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
-    expand tildes and globs in each root pattern and expand just tildes in each non-root pattern.
-    The idea is that non-root patterns may be regular expressions or other pattern styles containing
-    "*" that borgmatic should not expand as a shell glob.
-
-    Return all the resulting patterns as a tuple.
-
-    If a set of paths are given to skip, then don't expand any patterns matching them.
-    '''
-    if patterns is None:
-        return ()
-
-    return tuple(
-        itertools.chain.from_iterable(
-            (
-                (
-                    borgmatic.borg.pattern.Pattern(
-                        expanded_path,
-                        pattern.type,
-                        pattern.style,
-                        pattern.device,
-                        pattern.source,
-                    )
-                    for expanded_path in expand_directory(pattern.path, working_directory)
-                )
-                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-                and pattern.path not in (skip_paths or ())
-                else (
-                    borgmatic.borg.pattern.Pattern(
-                        os.path.expanduser(pattern.path),
-                        pattern.type,
-                        pattern.style,
-                        pattern.device,
-                        pattern.source,
-                    ),
-                )
-            )
-            for pattern in patterns
-        )
+    borgmatic_manifest_path = os.path.expanduser(
+        os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
     )
 
+    if not os.path.exists(borgmatic_manifest_path):
+        os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True)
 
-def device_map_patterns(patterns, working_directory=None):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
-    determine the identifier for the device on which the pattern's path resides—or None if the path
-    doesn't exist or is from a non-root pattern. Return an updated sequence of patterns with the
-    device field populated. But if the device field is already set, don't bother setting it again.
-
-    This is handy for determining whether two different pattern paths are on the same filesystem
-    (have the same device identifier).
-    '''
-    return tuple(
-        borgmatic.borg.pattern.Pattern(
-            pattern.path,
-            pattern.type,
-            pattern.style,
-            device=pattern.device
-            or (
-                os.stat(full_path).st_dev
-                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-                and os.path.exists(full_path)
-                else None
-            ),
-            source=pattern.source,
+    with open(borgmatic_manifest_path, 'w') as config_list_file:
+        json.dump(
+            {
+                'borgmatic_version': importlib.metadata.version('borgmatic'),
+                'config_paths': config_paths,
+            },
+            config_list_file,
         )
-        for pattern in patterns
-        for full_path in (os.path.join(working_directory or '', pattern.path),)
-    )
-
-
-def deduplicate_patterns(patterns):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
-    root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
-    "/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
-    modification.
-
-    The one exception to deduplication is two paths are on different filesystems (devices). In that
-    case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
-    one_file_system option is true).
-
-    The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
-    child patterns, because it will naturally spider the contents of the parent pattern's path. And
-    there are cases where Borg coming across the same file twice will result in duplicate reads and
-    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
-    Borg.
-    '''
-    deduplicated = {}  # Use just the keys as an ordered set.
-
-    for pattern in patterns:
-        if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
-            deduplicated[pattern] = True
-            continue
-
-        parents = pathlib.PurePath(pattern.path).parents
-
-        # If another directory in the given list is a parent of current directory (even n levels up)
-        # and both are on the same filesystem, then the current directory is a duplicate.
-        for other_pattern in patterns:
-            if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
-                continue
-
-            if any(
-                pathlib.PurePath(other_pattern.path) == parent
-                and pattern.device is not None
-                and other_pattern.device == pattern.device
-                for parent in parents
-            ):
-                break
-        else:
-            deduplicated[pattern] = True
-
-    return tuple(deduplicated.keys())
-
-
-def process_patterns(patterns, working_directory, skip_expand_paths=None):
-    '''
-    Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
-    "root" patterns, returning the resulting root and non-root patterns as a list.
-
-    If any paths are given to skip, don't expand them.
-    '''
-    skip_paths = set(skip_expand_paths or ())
-
-    return list(
-        deduplicate_patterns(
-            device_map_patterns(
-                expand_patterns(
-                    patterns,
-                    working_directory=working_directory,
-                    skip_paths=skip_paths,
-                )
-            )
-        )
-    )
 
 
 def run_create(
@@ -272,6 +48,7 @@ def run_create(
     repository,
     config,
     config_paths,
+    hook_context,
     local_borg_version,
     create_arguments,
     global_arguments,
@@ -289,58 +66,67 @@ def run_create(
     ):
         return
 
-    logger.info(f'Creating archive{dry_run_label}')
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
+    borgmatic.hooks.command.execute_hook(
+        config.get('before_backup'),
+        config.get('umask'),
+        config_filename,
+        'pre-backup',
+        global_arguments.dry_run,
+        **hook_context,
+    )
+    logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}')
+    borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+        'remove_data_source_dumps',
+        config,
+        repository['path'],
+        borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+        global_arguments.dry_run,
+    )
+    active_dumps = borgmatic.hooks.dispatch.call_hooks(
+        'dump_data_sources',
+        config,
+        repository['path'],
+        borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+        global_arguments.dry_run,
+    )
+    if config.get('store_config_files', True):
+        create_borgmatic_manifest(
             config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )
-        patterns = process_patterns(collect_patterns(config), working_directory)
-        active_dumps = borgmatic.hooks.dispatch.call_hooks(
-            'dump_data_sources',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             config_paths,
-            borgmatic_runtime_directory,
-            patterns,
             global_arguments.dry_run,
         )
+    stream_processes = [process for processes in active_dumps.values() for process in processes]
 
-        # Process the patterns again in case any data source hooks updated them. Without this step,
-        # we could end up with duplicate paths that cause Borg to hang when it tries to read from
-        # the same named pipe twice.
-        patterns = process_patterns(patterns, working_directory, skip_expand_paths=config_paths)
-        stream_processes = [process for processes in active_dumps.values() for process in processes]
+    json_output = borgmatic.borg.create.create_archive(
+        global_arguments.dry_run,
+        repository['path'],
+        config,
+        config_paths,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+        progress=create_arguments.progress,
+        stats=create_arguments.stats,
+        json=create_arguments.json,
+        list_files=create_arguments.list_files,
+        stream_processes=stream_processes,
+    )
+    if json_output:
+        yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
 
-        json_output = borgmatic.borg.create.create_archive(
-            global_arguments.dry_run,
-            repository['path'],
-            config,
-            patterns,
-            local_borg_version,
-            global_arguments,
-            borgmatic_runtime_directory,
-            local_path=local_path,
-            remote_path=remote_path,
-            progress=create_arguments.progress,
-            stats=create_arguments.stats,
-            json=create_arguments.json,
-            list_files=create_arguments.list_files,
-            stream_processes=stream_processes,
-        )
-
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
-
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )
+    borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+        'remove_data_source_dumps',
+        config,
+        config_filename,
+        borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+        global_arguments.dry_run,
+    )
+    borgmatic.hooks.command.execute_hook(
+        config.get('after_backup'),
+        config.get('umask'),
+        config_filename,
+        'post-backup',
+        global_arguments.dry_run,
+        **hook_context,
+    )

borgmatic/actions/delete.py

@@ -1,50 +0,0 @@
-import logging
-
-import borgmatic.actions.arguments
-import borgmatic.borg.delete
-import borgmatic.borg.repo_delete
-import borgmatic.borg.repo_list
-
-logger = logging.getLogger(__name__)
-
-
-def run_delete(
-    repository,
-    config,
-    local_borg_version,
-    delete_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "delete" action for the given repository and archive(s).
-    '''
-    if delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, delete_arguments.repository
-    ):
-        logger.answer('Deleting archives')
-
-        archive_name = (
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                delete_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            )
-            if delete_arguments.archive
-            else None
-        )
-
-        borgmatic.borg.delete.delete_archives(
-            repository,
-            config,
-            local_borg_version,
-            borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
-            global_arguments,
-            local_path,
-            remote_path,
-        )

borgmatic/actions/export_key.py

@@ -21,7 +21,7 @@ def run_export_key(
     if export_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, export_arguments.repository
     ):
-        logger.info('Exporting repository key')
+        logger.info(f'{repository.get("label", repository["path"])}: Exporting repository key')
         borgmatic.borg.export_key.export_key(
             repository['path'],
             config,

borgmatic/actions/export_tar.py

@@ -1,7 +1,7 @@
 import logging
 
 import borgmatic.borg.export_tar
-import borgmatic.borg.repo_list
+import borgmatic.borg.rlist
 import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
@@ -22,11 +22,13 @@ def run_export_tar(
     if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, export_tar_arguments.repository
     ):
-        logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file')
+        logger.info(
+            f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file'
+        )
         borgmatic.borg.export_tar.export_tar_archive(
             global_arguments.dry_run,
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
+            borgmatic.borg.rlist.resolve_archive_name(
                 repository['path'],
                 export_tar_arguments.archive,
                 config,

borgmatic/actions/extract.py

@@ -1,7 +1,7 @@
 import logging
 
 import borgmatic.borg.extract
-import borgmatic.borg.repo_list
+import borgmatic.borg.rlist
 import borgmatic.config.validate
 import borgmatic.hooks.command
 
@@ -12,6 +12,7 @@ def run_extract(
     config_filename,
     repository,
     config,
+    hook_context,
     local_borg_version,
     extract_arguments,
     global_arguments,
@@ -21,14 +22,24 @@ def run_extract(
     '''
     Run the "extract" action for the given repository.
     '''
+    borgmatic.hooks.command.execute_hook(
+        config.get('before_extract'),
+        config.get('umask'),
+        config_filename,
+        'pre-extract',
+        global_arguments.dry_run,
+        **hook_context,
+    )
     if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, extract_arguments.repository
     ):
-        logger.info(f'Extracting archive {extract_arguments.archive}')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Extracting archive {extract_arguments.archive}'
+        )
         borgmatic.borg.extract.extract_archive(
             global_arguments.dry_run,
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
+            borgmatic.borg.rlist.resolve_archive_name(
                 repository['path'],
                 extract_arguments.archive,
                 config,
@@ -47,3 +58,11 @@ def run_extract(
             strip_components=extract_arguments.strip_components,
             progress=extract_arguments.progress,
         )
+    borgmatic.hooks.command.execute_hook(
+        config.get('after_extract'),
+        config.get('umask'),
+        config_filename,
+        'post-extract',
+        global_arguments.dry_run,
+        **hook_context,
+    )

borgmatic/actions/import_key.py

@@ -1,33 +0,0 @@
-import logging
-
-import borgmatic.borg.import_key
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_import_key(
-    repository,
-    config,
-    local_borg_version,
-    import_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "key import" action for the given repository.
-    '''
-    if import_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, import_arguments.repository
-    ):
-        logger.info('Importing repository key')
-        borgmatic.borg.import_key.import_key(
-            repository['path'],
-            config,
-            local_borg_version,
-            import_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/info.py

@@ -3,7 +3,7 @@ import logging
 import borgmatic.actions.arguments
 import borgmatic.actions.json
 import borgmatic.borg.info
-import borgmatic.borg.repo_list
+import borgmatic.borg.rlist
 import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
@@ -27,8 +27,10 @@ def run_info(
         repository, info_arguments.repository
     ):
         if not info_arguments.json:
-            logger.answer('Displaying archive summary information')
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+            logger.answer(
+                f'{repository.get("label", repository["path"])}: Displaying archive summary information'
+            )
+        archive_name = borgmatic.borg.rlist.resolve_archive_name(
             repository['path'],
             info_arguments.archive,
             config,

borgmatic/actions/list.py

@@ -27,11 +27,11 @@ def run_list(
     ):
         if not list_arguments.json:
             if list_arguments.find_paths:  # pragma: no cover
-                logger.answer('Searching archives')
+                logger.answer(f'{repository.get("label", repository["path"])}: Searching archives')
             elif not list_arguments.archive:  # pragma: no cover
-                logger.answer('Listing archives')
+                logger.answer(f'{repository.get("label", repository["path"])}: Listing archives')
 
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+        archive_name = borgmatic.borg.rlist.resolve_archive_name(
             repository['path'],
             list_arguments.archive,
             config,

borgmatic/actions/mount.py

@@ -1,7 +1,7 @@
 import logging
 
 import borgmatic.borg.mount
-import borgmatic.borg.repo_list
+import borgmatic.borg.rlist
 import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
@@ -23,13 +23,15 @@ def run_mount(
         repository, mount_arguments.repository
     ):
         if mount_arguments.archive:
-            logger.info(f'Mounting archive {mount_arguments.archive}')
+            logger.info(
+                f'{repository.get("label", repository["path"])}: Mounting archive {mount_arguments.archive}'
+            )
         else:  # pragma: nocover
-            logger.info('Mounting repository')
+            logger.info(f'{repository.get("label", repository["path"])}: Mounting repository')
 
         borgmatic.borg.mount.mount_archive(
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
+            borgmatic.borg.rlist.resolve_archive_name(
                 repository['path'],
                 mount_arguments.archive,
                 config,

borgmatic/actions/prune.py

@@ -11,6 +11,7 @@ def run_prune(
     config_filename,
     repository,
     config,
+    hook_context,
     local_borg_version,
     prune_arguments,
     global_arguments,
@@ -26,7 +27,15 @@ def run_prune(
     ):
         return
 
-    logger.info(f'Pruning archives{dry_run_label}')
+    borgmatic.hooks.command.execute_hook(
+        config.get('before_prune'),
+        config.get('umask'),
+        config_filename,
+        'pre-prune',
+        global_arguments.dry_run,
+        **hook_context,
+    )
+    logger.info(f'{repository.get("label", repository["path"])}: Pruning archives{dry_run_label}')
     borgmatic.borg.prune.prune_archives(
         global_arguments.dry_run,
         repository['path'],
@@ -37,3 +46,11 @@ def run_prune(
         local_path=local_path,
         remote_path=remote_path,
     )
+    borgmatic.hooks.command.execute_hook(
+        config.get('after_prune'),
+        config.get('umask'),
+        config_filename,
+        'post-prune',
+        global_arguments.dry_run,
+        **hook_context,
+    )

borgmatic/actions/rcreate.py

@@ -0,0 +1,41 @@
+import logging
+
+import borgmatic.borg.rcreate
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_rcreate(
+    repository,
+    config,
+    local_borg_version,
+    rcreate_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "rcreate" action for the given repository.
+    '''
+    if rcreate_arguments.repository and not borgmatic.config.validate.repositories_match(
+        repository, rcreate_arguments.repository
+    ):
+        return
+
+    logger.info(f'{repository.get("label", repository["path"])}: Creating repository')
+    borgmatic.borg.rcreate.create_repository(
+        global_arguments.dry_run,
+        repository['path'],
+        config,
+        local_borg_version,
+        global_arguments,
+        rcreate_arguments.encryption_mode,
+        rcreate_arguments.source_repository,
+        rcreate_arguments.copy_crypt_key,
+        rcreate_arguments.append_only,
+        rcreate_arguments.storage_quota,
+        rcreate_arguments.make_parent_dirs,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/recreate.py

@@ -1,53 +0,0 @@
-import logging
-
-import borgmatic.borg.recreate
-import borgmatic.config.validate
-from borgmatic.actions.create import collect_patterns, process_patterns
-
-logger = logging.getLogger(__name__)
-
-
-def run_recreate(
-    repository,
-    config,
-    local_borg_version,
-    recreate_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "recreate" action for the given repository.
-    '''
-    if recreate_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, recreate_arguments.repository
-    ):
-        if recreate_arguments.archive:
-            logger.answer(f'Recreating archive {recreate_arguments.archive}')
-        else:
-            logger.answer('Recreating repository')
-
-        # Collect and process patterns.
-        processed_patterns = process_patterns(
-            collect_patterns(config), borgmatic.config.paths.get_working_directory(config)
-        )
-
-        borgmatic.borg.recreate.recreate_archive(
-            repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                recreate_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            config,
-            local_borg_version,
-            recreate_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            patterns=processed_patterns,
-        )

borgmatic/actions/repo_create.py

@@ -1,41 +0,0 @@
-import logging
-
-import borgmatic.borg.repo_create
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_repo_create(
-    repository,
-    config,
-    local_borg_version,
-    repo_create_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "repo-create" action for the given repository.
-    '''
-    if repo_create_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, repo_create_arguments.repository
-    ):
-        return
-
-    logger.info('Creating repository')
-    borgmatic.borg.repo_create.create_repository(
-        global_arguments.dry_run,
-        repository['path'],
-        config,
-        local_borg_version,
-        global_arguments,
-        repo_create_arguments.encryption_mode,
-        repo_create_arguments.source_repository,
-        repo_create_arguments.copy_crypt_key,
-        repo_create_arguments.append_only,
-        repo_create_arguments.storage_quota,
-        repo_create_arguments.make_parent_dirs,
-        local_path=local_path,
-        remote_path=remote_path,
-    )

borgmatic/actions/repo_delete.py

@@ -1,35 +0,0 @@
-import logging
-
-import borgmatic.borg.repo_delete
-
-logger = logging.getLogger(__name__)
-
-
-def run_repo_delete(
-    repository,
-    config,
-    local_borg_version,
-    repo_delete_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "repo-delete" action for the given repository.
-    '''
-    if repo_delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_delete_arguments.repository
-    ):
-        logger.answer(
-            'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else '')
-        )
-
-        borgmatic.borg.repo_delete.delete_repository(
-            repository,
-            config,
-            local_borg_version,
-            repo_delete_arguments,
-            global_arguments,
-            local_path,
-            remote_path,
-        )

borgmatic/actions/repo_info.py

@@ -1,40 +0,0 @@
-import logging
-
-import borgmatic.actions.json
-import borgmatic.borg.repo_info
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_repo_info(
-    repository,
-    config,
-    local_borg_version,
-    repo_info_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "repo-info" action for the given repository.
-
-    If repo_info_arguments.json is True, yield the JSON output from the info for the repository.
-    '''
-    if repo_info_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_info_arguments.repository
-    ):
-        if not repo_info_arguments.json:
-            logger.answer('Displaying repository summary information')
-
-        json_output = borgmatic.borg.repo_info.display_repository_info(
-            repository['path'],
-            config,
-            local_borg_version,
-            repo_info_arguments=repo_info_arguments,
-            global_arguments=global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/restore.py

@@ -1,156 +1,67 @@
-import collections
+import copy
 import logging
 import os
-import pathlib
-import shutil
-import tempfile
 
 import borgmatic.borg.extract
 import borgmatic.borg.list
 import borgmatic.borg.mount
-import borgmatic.borg.repo_list
-import borgmatic.config.paths
+import borgmatic.borg.rlist
+import borgmatic.borg.state
 import borgmatic.config.validate
-import borgmatic.hooks.data_source.dump
 import borgmatic.hooks.dispatch
+import borgmatic.hooks.dump
 
 logger = logging.getLogger(__name__)
 
 
-UNSPECIFIED = object()
+UNSPECIFIED_HOOK = object()
 
 
-Dump = collections.namedtuple(
-    'Dump',
-    ('hook_name', 'data_source_name', 'hostname', 'port'),
-    defaults=('localhost', None),
-)
-
-
-def dumps_match(first, second, default_port=None):
-    '''
-    Compare two Dump instances for equality while supporting a field value of UNSPECIFIED, which
-    indicates that the field should match any value. If a default port is given, then consider any
-    dump having that port to match with a dump having a None port.
-    '''
-    for field_name in first._fields:
-        first_value = getattr(first, field_name)
-        second_value = getattr(second, field_name)
-
-        if default_port is not None and field_name == 'port':
-            if first_value == default_port and second_value is None:
-                continue
-            if second_value == default_port and first_value is None:
-                continue
-
-        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:
-            continue
-
-        if first_value != second_value:
-            return False
-
-    return True
-
-
-def render_dump_metadata(dump):
-    '''
-    Given a Dump instance, make a display string describing it for use in log messages.
-    '''
-    name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
-    hostname = dump.hostname or UNSPECIFIED
-    port = None if dump.port is UNSPECIFIED else dump.port
-
-    if port:
-        metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
-    else:
-        metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
-
-    if dump.hook_name not in (None, UNSPECIFIED):
-        return f'{metadata} ({dump.hook_name})'
-
-    return metadata
-
-
-def get_configured_data_source(config, restore_dump):
-    '''
-    Search in the given configuration dict for dumps corresponding to the given dump to restore. If
-    there are multiple matches, error.
-
-    Return the found data source as a data source configuration dict or None if not found.
-    '''
-    try:
-        hooks_to_search = {restore_dump.hook_name: config[restore_dump.hook_name]}
-    except KeyError:
-        return None
-
-    matching_dumps = tuple(
-        hook_data_source
-        for (hook_name, hook_config) in hooks_to_search.items()
-        for hook_data_source in hook_config
-        for default_port in (
-            borgmatic.hooks.dispatch.call_hook(
-                function_name='get_default_port',
-                config=config,
-                hook_name=hook_name,
-            ),
-        )
-        if dumps_match(
-            Dump(
-                hook_name,
-                hook_data_source.get('name'),
-                hook_data_source.get('hostname', 'localhost'),
-                hook_data_source.get('port'),
-            ),
-            restore_dump,
-            default_port,
-        )
-    )
-
-    if not matching_dumps:
-        return None
-
-    if len(matching_dumps) > 1:
-        raise ValueError(
-            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured'
-        )
-
-    return matching_dumps[0]
-
-
-def strip_path_prefix_from_extracted_dump_destination(
-    destination_path, borgmatic_runtime_directory
+def get_configured_data_source(
+    config,
+    archive_data_source_names,
+    hook_name,
+    data_source_name,
+    configuration_data_source_name=None,
 ):
     '''
-    Directory-format dump files get extracted into a temporary directory containing a path prefix
-    that depends how the files were stored in the archive. So, given the destination path where the
-    dump was extracted and the borgmatic runtime directory, move the dump files such that the
-    restore doesn't have to deal with that varying path prefix.
+    Find the first data source with the given hook name and data source name in the configuration
+    dict and the given archive data source names dict (from hook name to data source names contained
+    in a particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all data
+    source hooks for the named data source. If a configuration data source name is given, use that
+    instead of the data source name to lookup the data source in the given hooks configuration.
 
-    For instance, if the dump was extracted to:
-
-      /run/user/0/borgmatic/tmp1234/borgmatic/postgresql_databases/test/...
-
-    or:
-
-      /run/user/0/borgmatic/tmp1234/root/.borgmatic/postgresql_databases/test/...
-
-    then this function moves it to:
-
-      /run/user/0/borgmatic/postgresql_databases/test/...
+    Return the found data source as a tuple of (found hook name, data source configuration dict) or
+    (None, None) if not found.
     '''
-    for subdirectory_path, _, _ in os.walk(destination_path):
-        databases_directory = os.path.basename(subdirectory_path)
+    if not configuration_data_source_name:
+        configuration_data_source_name = data_source_name
 
-        if not databases_directory.endswith('_databases'):
-            continue
+    if hook_name == UNSPECIFIED_HOOK:
+        hooks_to_search = {
+            hook_name: value
+            for (hook_name, value) in config.items()
+            if hook_name in borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES
+        }
+    else:
+        try:
+            hooks_to_search = {hook_name: config[hook_name]}
+        except KeyError:
+            return (None, None)
 
-        shutil.move(
-            subdirectory_path, os.path.join(borgmatic_runtime_directory, databases_directory)
-        )
-        break
+    return next(
+        (
+            (name, hook_data_source)
+            for (name, hook) in hooks_to_search.items()
+            for hook_data_source in hook
+            if hook_data_source['name'] == configuration_data_source_name
+            and data_source_name in archive_data_source_names.get(name, [])
+        ),
+        (None, None),
+    )
 
 
-def restore_single_dump(
+def restore_single_data_source(
     repository,
     config,
     local_borg_version,
@@ -161,78 +72,55 @@ def restore_single_dump(
     hook_name,
     data_source,
     connection_params,
-    borgmatic_runtime_directory,
-):
+):  # pragma: no cover
     '''
     Given (among other things) an archive name, a data source hook name, the hostname, port,
     username/password as connection params, and a configured data source configuration dict, restore
     that data source from the archive.
     '''
-    dump_metadata = render_dump_metadata(
-        Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'))
+    logger.info(
+        f'{repository.get("label", repository["path"])}: Restoring data source {data_source["name"]}'
     )
 
-    logger.info(f'Restoring data source {dump_metadata}')
-
-    dump_patterns = borgmatic.hooks.dispatch.call_hooks(
-        'make_data_source_dump_patterns',
+    dump_pattern = borgmatic.hooks.dispatch.call_hooks(
+        'make_data_source_dump_pattern',
         config,
-        borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-        borgmatic_runtime_directory,
+        repository['path'],
+        borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
         data_source['name'],
-    )[hook_name.split('_databases', 1)[0]]
+    )[hook_name]
 
-    destination_path = (
-        tempfile.mkdtemp(dir=borgmatic_runtime_directory)
-        if data_source.get('format') == 'directory'
-        else None
+    # Kick off a single data source extract to stdout.
+    extract_process = borgmatic.borg.extract.extract_archive(
+        dry_run=global_arguments.dry_run,
+        repository=repository['path'],
+        archive=archive_name,
+        paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
+        config=config,
+        local_borg_version=local_borg_version,
+        global_arguments=global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+        destination_path='/',
+        # A directory format dump isn't a single file, and therefore can't extract
+        # to stdout. In this case, the extract_process return value is None.
+        extract_to_stdout=bool(data_source.get('format') != 'directory'),
     )
 
-    try:
-        # Kick off a single data source extract. If using a directory format, extract to a temporary
-        # directory. Otherwise extract the single dump file to stdout.
-        extract_process = borgmatic.borg.extract.extract_archive(
-            dry_run=global_arguments.dry_run,
-            repository=repository['path'],
-            archive=archive_name,
-            paths=[
-                borgmatic.hooks.data_source.dump.convert_glob_patterns_to_borg_pattern(
-                    dump_patterns
-                )
-            ],
-            config=config,
-            local_borg_version=local_borg_version,
-            global_arguments=global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            destination_path=destination_path,
-            # A directory format dump isn't a single file, and therefore can't extract
-            # to stdout. In this case, the extract_process return value is None.
-            extract_to_stdout=bool(data_source.get('format') != 'directory'),
-        )
-
-        if destination_path and not global_arguments.dry_run:
-            strip_path_prefix_from_extracted_dump_destination(
-                destination_path, borgmatic_runtime_directory
-            )
-    finally:
-        if destination_path and not global_arguments.dry_run:
-            shutil.rmtree(destination_path, ignore_errors=True)
-
     # Run a single data source restore, consuming the extract stdout (if any).
-    borgmatic.hooks.dispatch.call_hook(
+    borgmatic.hooks.dispatch.call_hooks(
         function_name='restore_data_source_dump',
         config=config,
-        hook_name=hook_name,
+        log_prefix=repository['path'],
+        hook_names=[hook_name],
         data_source=data_source,
         dry_run=global_arguments.dry_run,
         extract_process=extract_process,
         connection_params=connection_params,
-        borgmatic_runtime_directory=borgmatic_runtime_directory,
     )
 
 
-def collect_dumps_from_archive(
+def collect_archive_data_source_names(
     repository,
     archive,
     config,
@@ -240,21 +128,18 @@ def collect_dumps_from_archive(
     global_arguments,
     local_path,
     remote_path,
-    borgmatic_runtime_directory,
 ):
     '''
     Given a local or remote repository path, a resolved archive name, a configuration dict, the
-    local Borg version, global arguments an argparse.Namespace, local and remote Borg paths, and the
-    borgmatic runtime directory, query the archive for the names of data sources dumps it contains
-    and return them as a set of Dump instances.
+    local Borg version, global_arguments an argparse.Namespace, and local and remote Borg paths,
+    query the archive for the names of data sources it contains as dumps and return them as a dict
+    from hook name to a sequence of data source names.
     '''
-    borgmatic_source_directory = str(
-        pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
-    )
-
-    # Probe for the data source dumps in multiple locations, as the default location has moved to
-    # the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
-    # we still want to support reading dumps from previously created archives as well.
+    borgmatic_source_directory = os.path.expanduser(
+        config.get(
+            'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+        )
+    ).lstrip('/')
     dump_paths = borgmatic.borg.list.capture_archive_listing(
         repository,
         archive,
@@ -262,162 +147,108 @@ def collect_dumps_from_archive(
         local_borg_version,
         global_arguments,
         list_paths=[
-            'sh:'
-            + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
-                base_directory, '*_databases/*/*'
-            )
-            for base_directory in (
-                'borgmatic',
-                borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
-                borgmatic_source_directory.lstrip('/'),
+            os.path.expanduser(
+                borgmatic.hooks.dump.make_data_source_dump_path(borgmatic_source_directory, pattern)
             )
+            for pattern in ('*_databases/*/*',)
         ],
         local_path=local_path,
         remote_path=remote_path,
     )
 
-    # Parse the paths of dumps found in the archive to get their respective dump metadata.
-    dumps_from_archive = set()
+    # Determine the data source names corresponding to the dumps found in the archive and
+    # add them to restore_names.
+    archive_data_source_names = {}
 
     for dump_path in dump_paths:
-        if not dump_path:
-            continue
-
-        # Probe to find the base directory that's at the start of the dump path.
-        for base_directory in (
-            'borgmatic',
-            borgmatic_runtime_directory,
-            borgmatic_source_directory,
-        ):
-            try:
-                (hook_name, host_and_port, data_source_name) = dump_path.split(
-                    base_directory + os.path.sep, 1
-                )[1].split(os.path.sep)[0:3]
-            except (ValueError, IndexError):
-                continue
-
-            parts = host_and_port.split(':', 1)
-
-            if len(parts) == 1:
-                parts += (None,)
-
-            (hostname, port) = parts
-
-            try:
-                port = int(port)
-            except (ValueError, TypeError):
-                port = None
-
-            dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port))
-
-            # We've successfully parsed the dump path, so need to probe any further.
-            break
-        else:
+        try:
+            (hook_name, _, data_source_name) = dump_path.split(
+                borgmatic_source_directory + os.path.sep, 1
+            )[1].split(os.path.sep)[0:3]
+        except (ValueError, IndexError):
             logger.warning(
-                f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
+                f'{repository}: Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
             )
-
-    return dumps_from_archive
-
-
-def get_dumps_to_restore(restore_arguments, dumps_from_archive):
-    '''
-    Given restore arguments as an argparse.Namespace instance indicating which dumps to restore and
-    a set of Dump instances representing the dumps found in an archive, return a set of specific
-    Dump instances from the archive to restore. As part of this, replace any Dump having a data
-    source name of "all" with multiple named Dump instances as appropriate.
-
-    Raise ValueError if any of the requested data source names cannot be found in the archive or if
-    there are multiple archive dump matches for a given requested dump.
-    '''
-    requested_dumps = (
-        {
-            Dump(
-                hook_name=(
-                    (
-                        restore_arguments.hook
-                        if restore_arguments.hook.endswith('_databases')
-                        else f'{restore_arguments.hook}_databases'
-                    )
-                    if restore_arguments.hook
-                    else UNSPECIFIED
-                ),
-                data_source_name=name,
-                hostname=restore_arguments.original_hostname or UNSPECIFIED,
-                port=restore_arguments.original_port,
-            )
-            for name in restore_arguments.data_sources or (UNSPECIFIED,)
-        }
-        if restore_arguments.hook
-        or restore_arguments.data_sources
-        or restore_arguments.original_hostname
-        or restore_arguments.original_port
-        else {
-            Dump(
-                hook_name=UNSPECIFIED,
-                data_source_name='all',
-                hostname=UNSPECIFIED,
-                port=UNSPECIFIED,
-            )
-        }
-    )
-    missing_dumps = set()
-    dumps_to_restore = set()
-
-    # If there's a requested "all" dump, add every dump from the archive to the dumps to restore.
-    if any(dump for dump in requested_dumps if dump.data_source_name == 'all'):
-        dumps_to_restore.update(dumps_from_archive)
-
-    # If any archive dump matches a requested dump, add the archive dump to the dumps to restore.
-    for requested_dump in requested_dumps:
-        if requested_dump.data_source_name == 'all':
-            continue
-
-        matching_dumps = tuple(
-            archive_dump
-            for archive_dump in dumps_from_archive
-            if dumps_match(requested_dump, archive_dump)
-        )
-
-        if len(matching_dumps) == 0:
-            missing_dumps.add(requested_dump)
-        elif len(matching_dumps) == 1:
-            dumps_to_restore.add(matching_dumps[0])
         else:
-            raise ValueError(
-                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.'
-            )
+            if data_source_name not in archive_data_source_names.get(hook_name, []):
+                archive_data_source_names.setdefault(hook_name, []).extend([data_source_name])
 
-    if missing_dumps:
-        rendered_dumps = ', '.join(
-            f'{render_dump_metadata(dump)}' for dump in sorted(missing_dumps)
-        )
-
-        raise ValueError(
-            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive"
-        )
-
-    return dumps_to_restore
+    return archive_data_source_names
 
 
-def ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored):
+def find_data_sources_to_restore(requested_data_source_names, archive_data_source_names):
     '''
-    Given a set of requested dumps to restore and a set of dumps actually restored, raise ValueError
-    if any requested dumps to restore weren't restored, indicating that they were missing from the
+    Given a sequence of requested data source names to restore and a dict of hook name to the names
+    of data sources found in an archive, return an expanded sequence of data source names to
+    restore, replacing "all" with actual data source names as appropriate.
+
+    Raise ValueError if any of the requested data source names cannot be found in the archive.
+    '''
+    # A map from data source hook name to the data source names to restore for that hook.
+    restore_names = (
+        {UNSPECIFIED_HOOK: requested_data_source_names}
+        if requested_data_source_names
+        else {UNSPECIFIED_HOOK: ['all']}
+    )
+
+    # If "all" is in restore_names, then replace it with the names of dumps found within the
+    # archive.
+    if 'all' in restore_names[UNSPECIFIED_HOOK]:
+        restore_names[UNSPECIFIED_HOOK].remove('all')
+
+        for hook_name, data_source_names in archive_data_source_names.items():
+            restore_names.setdefault(hook_name, []).extend(data_source_names)
+
+            # If a data source is to be restored as part of "all", then remove it from restore names
+            # so it doesn't get restored twice.
+            for data_source_name in data_source_names:
+                if data_source_name in restore_names[UNSPECIFIED_HOOK]:
+                    restore_names[UNSPECIFIED_HOOK].remove(data_source_name)
+
+    if not restore_names[UNSPECIFIED_HOOK]:
+        restore_names.pop(UNSPECIFIED_HOOK)
+
+    combined_restore_names = set(
+        name for data_source_names in restore_names.values() for name in data_source_names
+    )
+    combined_archive_data_source_names = set(
+        name
+        for data_source_names in archive_data_source_names.values()
+        for name in data_source_names
+    )
+
+    missing_names = sorted(set(combined_restore_names) - combined_archive_data_source_names)
+    if missing_names:
+        joined_names = ', '.join(f'"{name}"' for name in missing_names)
+        raise ValueError(
+            f"Cannot restore data source{'s' if len(missing_names) > 1 else ''} {joined_names} missing from archive"
+        )
+
+    return restore_names
+
+
+def ensure_data_sources_found(restore_names, remaining_restore_names, found_names):
+    '''
+    Given a dict from hook name to data source names to restore, a dict from hook name to remaining
+    data source names to restore, and a sequence of found (actually restored) data source names,
+    raise ValueError if requested data source to restore were missing from the archive and/or
     configuration.
     '''
-    if not dumps_actually_restored:
-        raise ValueError('No data source dumps were found to restore')
-
-    missing_dumps = sorted(
-        dumps_to_restore - dumps_actually_restored, key=lambda dump: dump.data_source_name
+    combined_restore_names = set(
+        name
+        for data_source_names in tuple(restore_names.values())
+        + tuple(remaining_restore_names.values())
+        for name in data_source_names
     )
 
-    if missing_dumps:
-        rendered_dumps = ', '.join(f'{render_dump_metadata(dump)}' for dump in missing_dumps)
+    if not combined_restore_names and not found_names:
+        raise ValueError('No data sources were found to restore')
 
+    missing_names = sorted(set(combined_restore_names) - set(found_names))
+    if missing_names:
+        joined_names = ', '.join(f'"{name}"' for name in missing_names)
         raise ValueError(
-            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration"
+            f"Cannot restore data source{'s' if len(missing_names) > 1 else ''} {joined_names} missing from borgmatic's configuration"
         )
 
 
@@ -434,79 +265,70 @@ def run_restore(
     Run the "restore" action for the given repository, but only if the repository matches the
     requested repository in restore arguments.
 
-    Raise ValueError if a configured data source could not be found to restore or there's no
-    matching dump in the archive.
+    Raise ValueError if a configured data source could not be found to restore.
     '''
     if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
         repository, restore_arguments.repository
     ):
         return
 
-    logger.info(f'Restoring data sources from archive {restore_arguments.archive}')
+    logger.info(
+        f'{repository.get("label", repository["path"])}: Restoring data sources from archive {restore_arguments.archive}'
+    )
 
-    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )
+    borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+        'remove_data_source_dumps',
+        config,
+        repository['path'],
+        borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+        global_arguments.dry_run,
+    )
 
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            repository['path'],
-            restore_arguments.archive,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        dumps_from_archive = collect_dumps_from_archive(
-            repository['path'],
-            archive_name,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-            borgmatic_runtime_directory,
-        )
-        dumps_to_restore = get_dumps_to_restore(restore_arguments, dumps_from_archive)
+    archive_name = borgmatic.borg.rlist.resolve_archive_name(
+        repository['path'],
+        restore_arguments.archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    archive_data_source_names = collect_archive_data_source_names(
+        repository['path'],
+        archive_name,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    restore_names = find_data_sources_to_restore(
+        restore_arguments.data_sources, archive_data_source_names
+    )
+    found_names = set()
+    remaining_restore_names = {}
+    connection_params = {
+        'hostname': restore_arguments.hostname,
+        'port': restore_arguments.port,
+        'username': restore_arguments.username,
+        'password': restore_arguments.password,
+        'restore_path': restore_arguments.restore_path,
+    }
 
-        dumps_actually_restored = set()
-        connection_params = {
-            'hostname': restore_arguments.hostname,
-            'port': restore_arguments.port,
-            'username': restore_arguments.username,
-            'password': restore_arguments.password,
-            'restore_path': restore_arguments.restore_path,
-        }
-
-        # Restore each dump.
-        for restore_dump in dumps_to_restore:
-            found_data_source = get_configured_data_source(
-                config,
-                restore_dump,
+    for hook_name, data_source_names in restore_names.items():
+        for data_source_name in data_source_names:
+            found_hook_name, found_data_source = get_configured_data_source(
+                config, archive_data_source_names, hook_name, data_source_name
             )
 
-            # For a dump that wasn't found via an exact match in the configuration, try to fallback
-            # to an "all" data source.
             if not found_data_source:
-                found_data_source = get_configured_data_source(
-                    config,
-                    Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
+                remaining_restore_names.setdefault(found_hook_name or hook_name, []).append(
+                    data_source_name
                 )
+                continue
 
-                if not found_data_source:
-                    continue
-
-                found_data_source = dict(found_data_source)
-                found_data_source['name'] = restore_dump.data_source_name
-
-            dumps_actually_restored.add(restore_dump)
-
-            restore_single_dump(
+            found_names.add(data_source_name)
+            restore_single_data_source(
                 repository,
                 config,
                 local_borg_version,
@@ -514,18 +336,45 @@ def run_restore(
                 local_path,
                 remote_path,
                 archive_name,
-                restore_dump.hook_name,
+                found_hook_name or hook_name,
                 dict(found_data_source, **{'schemas': restore_arguments.schemas}),
                 connection_params,
-                borgmatic_runtime_directory,
             )
 
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
-            config,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )
+    # For any data sources that weren't found via exact matches in the configuration, try to
+    # fallback to "all" entries.
+    for hook_name, data_source_names in remaining_restore_names.items():
+        for data_source_name in data_source_names:
+            found_hook_name, found_data_source = get_configured_data_source(
+                config, archive_data_source_names, hook_name, data_source_name, 'all'
+            )
 
-    ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored)
+            if not found_data_source:
+                continue
+
+            found_names.add(data_source_name)
+            data_source = copy.copy(found_data_source)
+            data_source['name'] = data_source_name
+
+            restore_single_data_source(
+                repository,
+                config,
+                local_borg_version,
+                global_arguments,
+                local_path,
+                remote_path,
+                archive_name,
+                found_hook_name or hook_name,
+                dict(data_source, **{'schemas': restore_arguments.schemas}),
+                connection_params,
+            )
+
+    borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+        'remove_data_source_dumps',
+        config,
+        repository['path'],
+        borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
+        global_arguments.dry_run,
+    )
+
+    ensure_data_sources_found(restore_names, remaining_restore_names, found_names)

borgmatic/actions/rinfo.py

@@ -0,0 +1,42 @@
+import logging
+
+import borgmatic.actions.json
+import borgmatic.borg.rinfo
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_rinfo(
+    repository,
+    config,
+    local_borg_version,
+    rinfo_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "rinfo" action for the given repository.
+
+    If rinfo_arguments.json is True, yield the JSON output from the info for the repository.
+    '''
+    if rinfo_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, rinfo_arguments.repository
+    ):
+        if not rinfo_arguments.json:
+            logger.answer(
+                f'{repository.get("label", repository["path"])}: Displaying repository summary information'
+            )
+
+        json_output = borgmatic.borg.rinfo.display_repository_info(
+            repository['path'],
+            config,
+            local_borg_version,
+            rinfo_arguments=rinfo_arguments,
+            global_arguments=global_arguments,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+        if json_output:
+            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/rlist.py

@@ -1,37 +1,37 @@
 import logging
 
 import borgmatic.actions.json
-import borgmatic.borg.repo_list
+import borgmatic.borg.rlist
 import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
 
-def run_repo_list(
+def run_rlist(
     repository,
     config,
     local_borg_version,
-    repo_list_arguments,
+    rlist_arguments,
     global_arguments,
     local_path,
     remote_path,
 ):
     '''
-    Run the "repo-list" action for the given repository.
+    Run the "rlist" action for the given repository.
 
-    If repo_list_arguments.json is True, yield the JSON output from listing the repository.
+    If rlist_arguments.json is True, yield the JSON output from listing the repository.
     '''
-    if repo_list_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_list_arguments.repository
+    if rlist_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, rlist_arguments.repository
     ):
-        if not repo_list_arguments.json:
-            logger.answer('Listing repository')
+        if not rlist_arguments.json:
+            logger.answer(f'{repository.get("label", repository["path"])}: Listing repository')
 
-        json_output = borgmatic.borg.repo_list.list_repository(
+        json_output = borgmatic.borg.rlist.list_repository(
             repository['path'],
             config,
             local_borg_version,
-            repo_list_arguments=repo_list_arguments,
+            rlist_arguments=rlist_arguments,
             global_arguments=global_arguments,
             local_path=local_path,
             remote_path=remote_path,

borgmatic/actions/transfer.py

@@ -17,7 +17,9 @@ def run_transfer(
     '''
     Run the "transfer" action for the given repository.
     '''
-    logger.info('Transferring archives to repository')
+    logger.info(
+        f'{repository.get("label", repository["path"])}: Transferring archives to repository'
+    )
     borgmatic.borg.transfer.transfer_archives(
         global_arguments.dry_run,
         repository['path'],

borgmatic/borg/borg.py

@@ -2,7 +2,6 @@ import logging
 import shlex
 
 import borgmatic.commands.arguments
-import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@@ -61,14 +60,13 @@ def run_arbitrary_borg(
         tuple(shlex.quote(part) for part in full_command),
         output_file=DO_NOT_CAPTURE,
         shell=True,
-        environment=dict(
+        extra_environment=dict(
             (environment.make_environment(config) or {}),
             **{
                 'BORG_REPO': repository_path,
                 'ARCHIVE': archive if archive else '',
             },
         ),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/break_lock.py

@@ -1,6 +1,5 @@
 import logging
 
-import borgmatic.config.paths
 from borgmatic.borg import environment, flags
 from borgmatic.execute import execute_command
 
@@ -34,10 +33,10 @@ def break_lock(
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
+    borg_environment = environment.make_environment(config)
     execute_command(
         full_command,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        extra_environment=borg_environment,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/change_passphrase.py

@@ -1,67 +0,0 @@
-import logging
-
-import borgmatic.config.paths
-import borgmatic.execute
-import borgmatic.logger
-from borgmatic.borg import environment, flags
-
-logger = logging.getLogger(__name__)
-
-
-def change_passphrase(
-    repository_path,
-    config,
-    local_borg_version,
-    change_passphrase_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, change
-    passphrase arguments, and optional local and remote Borg paths, change the repository passphrase
-    based on an interactive prompt.
-    '''
-    borgmatic.logger.add_custom_log_levels()
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
-
-    full_command = (
-        (local_path, 'key', 'change-passphrase')
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + flags.make_repository_flags(
-            repository_path,
-            local_borg_version,
-        )
-    )
-
-    if global_arguments.dry_run:
-        logger.info('Skipping change password (dry run)')
-        return
-
-    # If the original passphrase is set programmatically, then Borg won't prompt for a new one! So
-    # don't give Borg any passphrase, and it'll ask the user for both old and new ones.
-    config_without_passphrase = {
-        option_name: value
-        for (option_name, value) in config.items()
-        if option_name not in ('encryption_passphrase', 'encryption_passcommand')
-    }
-
-    borgmatic.execute.execute_command(
-        full_command,
-        output_file=borgmatic.execute.DO_NOT_CAPTURE,
-        output_log_level=logging.ANSWER,
-        environment=environment.make_environment(config_without_passphrase),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )
-
-    logger.answer(
-        f"{repository_path}: Don't forget to update your encryption_passphrase option (if needed)"
-    )

borgmatic/borg/check.py

@@ -2,8 +2,7 @@ import argparse
 import json
 import logging
 
-import borgmatic.config.paths
-from borgmatic.borg import environment, feature, flags, repo_info
+from borgmatic.borg import environment, feature, flags, rinfo
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
@@ -51,10 +50,10 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument
     return ()
 
 
-def make_check_name_flags(checks, archive_filter_flags):
+def make_check_flags(checks, archive_filter_flags):
     '''
-    Given parsed checks set and a sequence of flags to filter archives, transform the checks into
-    tuple of command-line check flags.
+    Given a parsed sequence of checks and a sequence of flags to filter archives, transform the
+    checks into tuple of command-line check flags.
 
     For example, given parsed checks of:
 
@@ -64,14 +63,18 @@ def make_check_name_flags(checks, archive_filter_flags):
 
         ('--repository-only',)
 
-    However, if both "repository" and "archives" are in checks, then omit the "only" flags from the
-    returned flags because Borg does both checks by default. Note that a "data" check only works
-    along with an "archives" check.
+    However, if both "repository" and "archives" are in checks, then omit them from the returned
+    flags because Borg does both checks by default. If "data" is in checks, that implies "archives".
     '''
-    data_flags = ('--verify-data',) if 'data' in checks else ()
+    if 'data' in checks:
+        data_flags = ('--verify-data',)
+        checks += ('archives',)
+    else:
+        data_flags = ()
+
     common_flags = (archive_filter_flags if 'archives' in checks else ()) + data_flags
 
-    if {'repository', 'archives'}.issubset(checks):
+    if {'repository', 'archives'}.issubset(set(checks)):
         return common_flags
 
     return (
@@ -91,7 +94,7 @@ def get_repository_id(
     '''
     try:
         return json.loads(
-            repo_info.display_repository_info(
+            rinfo.display_repository_info(
                 repository_path,
                 config,
                 local_borg_version,
@@ -131,59 +134,36 @@ def check_archives(
     if logger.isEnabledFor(logging.DEBUG):
         verbosity_flags = ('--debug', '--show-rc')
 
-    try:
-        repository_check_config = next(
-            check for check in config.get('checks', ()) if check.get('name') == 'repository'
-        )
-    except StopIteration:
-        repository_check_config = {}
+    full_command = (
+        (local_path, 'check')
+        + (('--repair',) if check_arguments.repair else ())
+        + make_check_flags(checks, archive_filter_flags)
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + verbosity_flags
+        + (('--progress',) if check_arguments.progress else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + flags.make_repository_flags(repository_path, local_borg_version)
+    )
 
-    max_duration = check_arguments.max_duration or repository_check_config.get('max_duration')
-
-    umask = config.get('umask')
+    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    if 'data' in checks:
-        checks.add('archives')
-
-    grouped_checks = (checks,)
-
-    # If max_duration is set, then archives and repository checks need to be run separately, as Borg
-    # doesn't support --max-duration along with an archives checks.
-    if max_duration and 'archives' in checks and 'repository' in checks:
-        checks.remove('repository')
-        grouped_checks = (checks, {'repository'})
-
-    for checks_subset in grouped_checks:
-        full_command = (
-            (local_path, 'check')
-            + (('--repair',) if check_arguments.repair else ())
-            + (
-                ('--max-duration', str(max_duration))
-                if max_duration and 'repository' in checks_subset
-                else ()
-            )
-            + make_check_name_flags(checks_subset, archive_filter_flags)
-            + (('--remote-path', remote_path) if remote_path else ())
-            + (('--umask', str(umask)) if umask else ())
-            + (('--log-json',) if global_arguments.log_json else ())
-            + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-            + verbosity_flags
-            + (('--progress',) if check_arguments.progress else ())
-            + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-            + flags.make_repository_flags(repository_path, local_borg_version)
-        )
 
+    # The Borg repair option triggers an interactive prompt, which won't work when output is
+    # captured. And progress messes with the terminal directly.
+    if check_arguments.repair or check_arguments.progress:
         execute_command(
             full_command,
-            # The Borg repair option triggers an interactive prompt, which won't work when output is
-            # captured. And progress messes with the terminal directly.
-            output_file=(
-                DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None
-            ),
-            environment=environment.make_environment(config),
-            working_directory=working_directory,
+            output_file=DO_NOT_CAPTURE,
+            extra_environment=borg_environment,
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
+        )
+    else:
+        execute_command(
+            full_command,
+            extra_environment=borg_environment,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/compact.py

@@ -1,6 +1,5 @@
 import logging
 
-import borgmatic.config.paths
 from borgmatic.borg import environment, flags
 from borgmatic.execute import execute_command
 
@@ -43,14 +42,13 @@ def compact_segments(
     )
 
     if dry_run:
-        logging.info('Skipping compact (dry run)')
+        logging.info(f'{repository_path}: Skipping compact (dry run)')
         return
 
     execute_command(
         full_command,
         output_log_level=logging.INFO,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/create.py

@@ -1,15 +1,13 @@
+import glob
 import itertools
 import logging
 import os
 import pathlib
 import stat
 import tempfile
-import textwrap
 
-import borgmatic.borg.pattern
-import borgmatic.config.paths
 import borgmatic.logger
-from borgmatic.borg import environment, feature, flags
+from borgmatic.borg import environment, feature, flags, state
 from borgmatic.execute import (
     DO_NOT_CAPTURE,
     execute_command,
@@ -20,42 +18,163 @@ from borgmatic.execute import (
 logger = logging.getLogger(__name__)
 
 
-def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None):
+def expand_directory(directory):
     '''
-    Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
-    temporary file in the given borgmatic runtime directory and return the file object so it can
-    continue to exist on disk as long as the caller needs it.
+    Given a directory path, expand any tilde (representing a user's home directory) and any globs
+    therein. Return a list of one or more resulting paths.
+    '''
+    expanded_directory = os.path.expanduser(directory)
 
-    If an optional open pattern file is given, append to it instead of making a new temporary file.
+    return glob.glob(expanded_directory) or [expanded_directory]
+
+
+def expand_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes and globs in each one. Return all the
+    resulting directories as a single flattened tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(expand_directory(directory) for directory in directories)
+    )
+
+
+def expand_home_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
+    Return the results as a tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(os.path.expanduser(directory) for directory in directories)
+
+
+def map_directories_to_devices(directories):
+    '''
+    Given a sequence of directories, return a map from directory to an identifier for the device on
+    which that directory resides or None if the path doesn't exist.
+
+    This is handy for determining whether two different directories are on the same filesystem (have
+    the same device identifier).
+    '''
+    return {
+        directory: os.stat(directory).st_dev if os.path.exists(directory) else None
+        for directory in directories
+    }
+
+
+def deduplicate_directories(directory_devices, additional_directory_devices):
+    '''
+    Given a map from directory to the identifier for the device on which that directory resides,
+    return the directories as a sorted tuple with all duplicate child directories removed. For
+    instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
+
+    The one exception to this rule is if two paths are on different filesystems (devices). In that
+    case, they won't get de-duplicated in case they both need to be passed to Borg (e.g. the
+    location.one_file_system option is true).
+
+    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
+    child directories, because it will naturally spider the contents of the parent directory. And
+    there are cases where Borg coming across the same file twice will result in duplicate reads and
+    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
+    Borg.
+
+    If any additional directory devices are given, also deduplicate against them, but don't include
+    them in the returned directories.
+    '''
+    deduplicated = set()
+    directories = sorted(directory_devices.keys())
+    additional_directories = sorted(additional_directory_devices.keys())
+    all_devices = {**directory_devices, **additional_directory_devices}
+
+    for directory in directories:
+        deduplicated.add(directory)
+        parents = pathlib.PurePath(directory).parents
+
+        # If another directory in the given list (or the additional list) is a parent of current
+        # directory (even n levels up) and both are on the same filesystem, then the current
+        # directory is a duplicate.
+        for other_directory in directories + additional_directories:
+            for parent in parents:
+                if (
+                    pathlib.PurePath(other_directory) == parent
+                    and all_devices[directory] is not None
+                    and all_devices[other_directory] == all_devices[directory]
+                ):
+                    if directory in deduplicated:
+                        deduplicated.remove(directory)
+                    break
+
+    return tuple(sorted(deduplicated))
+
+
+def write_pattern_file(patterns=None, sources=None, pattern_file=None):
+    '''
+    Given a sequence of patterns and an optional sequence of source directories, write them to a
+    named temporary file (with the source directories as additional roots) and return the file.
+    If an optional open pattern file is given, overwrite it instead of making a new temporary file.
     Return None if no patterns are provided.
     '''
-    if not patterns:
+    if not patterns and not sources:
         return None
 
-    if patterns_file is None:
-        patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
-        operation_name = 'Writing'
+    if pattern_file is None:
+        pattern_file = tempfile.NamedTemporaryFile('w')
     else:
-        patterns_file.write('\n')
-        operation_name = 'Appending'
+        pattern_file.seek(0)
 
-    patterns_output = '\n'.join(
-        f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}'
-        for pattern in patterns
+    pattern_file.write(
+        '\n'.join(tuple(patterns or ()) + tuple(f'R {source}' for source in (sources or [])))
     )
-    logger.debug(f'{operation_name} patterns to {patterns_file.name}:\n{patterns_output}')
+    pattern_file.flush()
 
-    patterns_file.write(patterns_output)
-    patterns_file.flush()
-
-    return patterns_file
+    return pattern_file
 
 
-def make_exclude_flags(config):
+def ensure_files_readable(*filename_lists):
     '''
-    Given a configuration dict with various exclude options, return the corresponding Borg flags as
-    a tuple.
+    Given a sequence of filename sequences, ensure that each filename is openable. This prevents
+    unreadable files from being passed to Borg, which in certain situations only warns instead of
+    erroring.
     '''
+    for file_object in itertools.chain.from_iterable(
+        filename_list for filename_list in filename_lists if filename_list
+    ):
+        open(file_object).close()
+
+
+def make_pattern_flags(config, pattern_filename=None):
+    '''
+    Given a configuration dict with a potential patterns_from option, and a filename containing any
+    additional patterns, return the corresponding Borg flags for those files as a tuple.
+    '''
+    pattern_filenames = tuple(config.get('patterns_from') or ()) + (
+        (pattern_filename,) if pattern_filename else ()
+    )
+
+    return tuple(
+        itertools.chain.from_iterable(
+            ('--patterns-from', pattern_filename) for pattern_filename in pattern_filenames
+        )
+    )
+
+
+def make_exclude_flags(config, exclude_filename=None):
+    '''
+    Given a configuration dict with various exclude options, and a filename containing any exclude
+    patterns, return the corresponding Borg flags as a tuple.
+    '''
+    exclude_filenames = tuple(config.get('exclude_from') or ()) + (
+        (exclude_filename,) if exclude_filename else ()
+    )
+    exclude_from_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-from', exclude_filename) for exclude_filename in exclude_filenames
+        )
+    )
     caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else ()
     if_present_flags = tuple(
         itertools.chain.from_iterable(
@@ -66,7 +185,13 @@ def make_exclude_flags(config):
     keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
     exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
 
-    return caches_flag + if_present_flags + keep_exclude_tags_flags + exclude_nodump_flags
+    return (
+        exclude_from_flags
+        + caches_flag
+        + if_present_flags
+        + keep_exclude_tags_flags
+        + exclude_nodump_flags
+    )
 
 
 def make_list_filter_flags(local_borg_version, dry_run):
@@ -90,14 +215,44 @@ def make_list_filter_flags(local_borg_version, dry_run):
         return f'{base_flags}-'
 
 
-def special_file(path, working_directory=None):
+def collect_borgmatic_source_directories(borgmatic_source_directory):
+    '''
+    Return a list of borgmatic-specific source directories used for state like database backups.
+    '''
+    if not borgmatic_source_directory:
+        borgmatic_source_directory = state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+
+    return (
+        [borgmatic_source_directory]
+        if os.path.exists(os.path.expanduser(borgmatic_source_directory))
+        else []
+    )
+
+
+ROOT_PATTERN_PREFIX = 'R '
+
+
+def pattern_root_directories(patterns=None):
+    '''
+    Given a sequence of patterns, parse out and return just the root directories.
+    '''
+    if not patterns:
+        return []
+
+    return [
+        pattern.split(ROOT_PATTERN_PREFIX, maxsplit=1)[1]
+        for pattern in patterns
+        if pattern.startswith(ROOT_PATTERN_PREFIX)
+    ]
+
+
+def special_file(path):
     '''
     Return whether the given path is a special file (character device, block device, or named pipe
-    / FIFO). If a working directory is given, take it into account when making the full path to
-    check.
+    / FIFO).
     '''
     try:
-        mode = os.stat(os.path.join(working_directory or '', path)).st_mode
+        mode = os.stat(path).st_mode
     except (FileNotFoundError, OSError):
         return False
 
@@ -117,100 +272,62 @@ def any_parent_directories(path, candidate_parents):
 
 
 def collect_special_file_paths(
-    dry_run,
-    create_command,
-    config,
-    local_path,
-    working_directory,
-    borgmatic_runtime_directory,
+    create_command, config, local_path, working_directory, borg_environment, skip_directories
 ):
     '''
-    Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
-    a working directory, and the borgmatic runtime directory, collect the paths for any special
-    files (character devices, block devices, and named pipes / FIFOs) that Borg would encounter
-    during a create. These are all paths that could cause Borg to hang if its --read-special flag is
-    used.
-
-    Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
-    its own special files there for database dumps and we don't want those omitted.
-
-    Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg
-    plans to backup, that means the user must have excluded the runtime directory (e.g. via
-    "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to
-    consume any database dumps and therefore borgmatic will hang when it tries to do so.
+    Given a Borg create command as a tuple, a configuration dict, a local Borg path, a working
+    directory, a dict of environment variables to pass to Borg, and a sequence of parent directories
+    to skip, collect the paths for any special files (character devices, block devices, and named
+    pipes / FIFOs) that Borg would encounter during a create. These are all paths that could cause
+    Borg to hang if its --read-special flag is used.
     '''
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
-    # files including any named pipe we've created. And omit "--filter" because that can break the
-    # paths output parsing below such that path lines no longer start with th expected "- ".
+    # files including any named pipe we've created.
     paths_output = execute_command_and_capture_output(
-        flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter')
+        tuple(argument for argument in create_command if argument != '--exclude-nodump')
         + ('--dry-run', '--list'),
         capture_stderr=True,
         working_directory=working_directory,
-        environment=environment.make_environment(config),
+        extra_environment=borg_environment,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
 
-    # These are all the individual files that Borg is planning to backup as determined by the Borg
-    # create dry run above.
     paths = tuple(
         path_line.split(' ', 1)[1]
         for path_line in paths_output.split('\n')
         if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
     )
 
-    # These are the subset of those files that contain the borgmatic runtime directory.
-    paths_containing_runtime_directory = {}
-
-    if os.path.exists(borgmatic_runtime_directory):
-        paths_containing_runtime_directory = {
-            path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
-        }
-
-        # If no paths to backup contain the runtime directory, it must've been excluded.
-        if not paths_containing_runtime_directory and not dry_run:
-            raise ValueError(
-                f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
-            )
-
     return tuple(
         path
         for path in paths
-        if special_file(path, working_directory)
-        if path not in paths_containing_runtime_directory
+        if special_file(path) and not any_parent_directories(path, skip_directories)
     )
 
 
-def check_all_root_patterns_exist(patterns):
+def check_all_source_directories_exist(source_directories):
     '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
-    paths exist. If any don't, raise an exception.
+    Given a sequence of source directories, check that they all exist. If any do not, raise an
+    exception.
     '''
-    missing_paths = [
-        pattern.path
-        for pattern in patterns
-        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-        if not os.path.exists(pattern.path)
+    missing_directories = [
+        source_directory
+        for source_directory in source_directories
+        if not all([os.path.exists(directory) for directory in expand_directory(source_directory)])
     ]
-
-    if missing_paths:
-        raise ValueError(
-            f"Source directories / root pattern paths do not exist: {', '.join(missing_paths)}"
-        )
-
-
-MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
+    if missing_directories:
+        raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
 
 
 def make_base_create_command(
     dry_run,
     repository_path,
     config,
-    patterns,
+    config_paths,
     local_borg_version,
     global_arguments,
-    borgmatic_runtime_directory,
+    borgmatic_source_directories,
     local_path='borg',
     remote_path=None,
     progress=False,
@@ -219,29 +336,46 @@ def make_base_create_command(
     stream_processes=None,
 ):
     '''
-    Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version,
-    global arguments as an argparse.Namespace instance, and a sequence of borgmatic source
-    directories, return a tuple of (base Borg create command flags, Borg create command positional
-    arguments, open pattern file handle).
+    Given vebosity/dry-run flags, a local or remote repository path, a configuration dict, a
+    sequence of loaded configuration paths, the local Borg version, global arguments as an
+    argparse.Namespace instance, and a sequence of borgmatic source directories, return a tuple of
+    (base Borg create command flags, Borg create command positional arguments, open pattern file
+    handle, open exclude file handle).
     '''
     if config.get('source_directories_must_exist', False):
-        check_all_root_patterns_exist(patterns)
+        check_all_source_directories_exist(config.get('source_directories'))
 
-    patterns_file = write_patterns_file(patterns, borgmatic_runtime_directory)
+    sources = deduplicate_directories(
+        map_directories_to_devices(
+            expand_directories(
+                tuple(config.get('source_directories', ()))
+                + borgmatic_source_directories
+                + tuple(config_paths if config.get('store_config_files', True) else ())
+            )
+        ),
+        additional_directory_devices=map_directories_to_devices(
+            expand_directories(pattern_root_directories(config.get('patterns')))
+        ),
+    )
+
+    ensure_files_readable(config.get('patterns_from'), config.get('exclude_from'))
+
+    pattern_file = (
+        write_pattern_file(config.get('patterns'), sources)
+        if config.get('patterns') or config.get('patterns_from')
+        else None
+    )
+    exclude_file = write_pattern_file(expand_home_directories(config.get('exclude_patterns')))
     checkpoint_interval = config.get('checkpoint_interval', None)
     checkpoint_volume = config.get('checkpoint_volume', None)
     chunker_params = config.get('chunker_params', None)
     compression = config.get('compression', None)
     upload_rate_limit = config.get('upload_rate_limit', None)
-    upload_buffer_size = config.get('upload_buffer_size', None)
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
     list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
     files_cache = config.get('files_cache')
-    archive_name_format = config.get(
-        'archive_name_format', flags.get_default_archive_name_format(local_borg_version)
-    )
+    archive_name_format = config.get('archive_name_format', flags.DEFAULT_ARCHIVE_NAME_FORMAT)
     extra_borg_options = config.get('extra_borg_options', {}).get('create', '')
 
     if feature.available(feature.Feature.ATIME, local_borg_version):
@@ -271,15 +405,14 @@ def make_base_create_command(
     create_flags = (
         tuple(local_path.split(' '))
         + ('create',)
-        + (('--patterns-from', patterns_file.name) if patterns_file else ())
-        + make_exclude_flags(config)
+        + make_pattern_flags(config, pattern_file.name if pattern_file else None)
+        + make_exclude_flags(config, exclude_file.name if exclude_file else None)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
         + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ())
         + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
         + upload_ratelimit_flags
-        + (('--upload-buffer', str(upload_buffer_size)) if upload_buffer_size else ())
-        + (('--one-file-system',) if config.get('one_file_system') else ())
+        + (('--one-file-system',) if config.get('one_file_system') or stream_processes else ())
         + numeric_ids_flags
         + atime_flags
         + (('--noctime',) if config.get('ctime') is False else ())
@@ -302,63 +435,53 @@ def make_base_create_command(
 
     create_positional_arguments = flags.make_repository_archive_flags(
         repository_path, archive_name_format, local_borg_version
-    )
+    ) + (sources if not pattern_file else ())
 
     # If database hooks are enabled (as indicated by streaming processes), exclude files that might
     # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
     if stream_processes and not config.get('read_special'):
         logger.warning(
-            'Ignoring configured "read_special" value of false, as true is needed for database hooks.'
+            f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
         )
-        working_directory = borgmatic.config.paths.get_working_directory(config)
+        try:
+            working_directory = os.path.expanduser(config.get('working_directory'))
+        except TypeError:
+            working_directory = None
 
-        logger.debug('Collecting special file paths')
+        borg_environment = environment.make_environment(config)
+
+        logger.debug(f'{repository_path}: Collecting special file paths')
         special_file_paths = collect_special_file_paths(
-            dry_run,
             create_flags + create_positional_arguments,
             config,
             local_path,
             working_directory,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
+            borg_environment,
+            skip_directories=borgmatic_source_directories,
         )
 
         if special_file_paths:
-            truncated_special_file_paths = textwrap.shorten(
-                ', '.join(special_file_paths),
-                width=MAX_SPECIAL_FILE_PATHS_LENGTH,
-                placeholder=' ...',
-            )
             logger.warning(
-                f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}'
+                f'{repository_path}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}'
             )
-            patterns_file = write_patterns_file(
-                tuple(
-                    borgmatic.borg.pattern.Pattern(
-                        special_file_path,
-                        borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
-                        borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                        source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
-                    )
-                    for special_file_path in special_file_paths
+            exclude_file = write_pattern_file(
+                expand_home_directories(
+                    tuple(config.get('exclude_patterns') or ()) + special_file_paths
                 ),
-                borgmatic_runtime_directory,
-                patterns_file=patterns_file,
+                pattern_file=exclude_file,
             )
+            create_flags += make_exclude_flags(config, exclude_file.name)
 
-            if '--patterns-from' not in create_flags:
-                create_flags += ('--patterns-from', patterns_file.name)
-
-    return (create_flags, create_positional_arguments, patterns_file)
+    return (create_flags, create_positional_arguments, pattern_file, exclude_file)
 
 
 def create_archive(
     dry_run,
     repository_path,
     config,
-    patterns,
+    config_paths,
     local_borg_version,
     global_arguments,
-    borgmatic_runtime_directory,
     local_path='borg',
     remote_path=None,
     progress=False,
@@ -368,7 +491,7 @@ def create_archive(
     stream_processes=None,
 ):
     '''
-    Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
+    Given vebosity/dry-run flags, a local or remote repository path, a configuration dict, a
     sequence of loaded configuration paths, the local Borg version, and global arguments as an
     argparse.Namespace instance, create a Borg archive and return Borg's JSON output (if any).
 
@@ -376,23 +499,26 @@ def create_archive(
     create command while also triggering the given processes to produce output.
     '''
     borgmatic.logger.add_custom_log_levels()
+    borgmatic_source_directories = expand_directories(
+        collect_borgmatic_source_directories(config.get('borgmatic_source_directory'))
+    )
 
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    (create_flags, create_positional_arguments, patterns_file) = make_base_create_command(
-        dry_run,
-        repository_path,
-        config,
-        patterns,
-        local_borg_version,
-        global_arguments,
-        borgmatic_runtime_directory,
-        local_path,
-        remote_path,
-        progress,
-        json,
-        list_files,
-        stream_processes,
+    (create_flags, create_positional_arguments, pattern_file, exclude_file) = (
+        make_base_create_command(
+            dry_run,
+            repository_path,
+            config,
+            config_paths,
+            local_borg_version,
+            global_arguments,
+            borgmatic_source_directories,
+            local_path,
+            remote_path,
+            progress,
+            json,
+            list_files,
+            stream_processes,
+        )
     )
 
     if json:
@@ -406,6 +532,13 @@ def create_archive(
     # the terminal directly.
     output_file = DO_NOT_CAPTURE if progress else None
 
+    try:
+        working_directory = os.path.expanduser(config.get('working_directory'))
+    except TypeError:
+        working_directory = None
+
+    borg_environment = environment.make_environment(config)
+
     create_flags += (
         (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
         + (('--stats',) if stats and not json and not dry_run else ())
@@ -422,7 +555,7 @@ def create_archive(
             output_log_level,
             output_file,
             working_directory=working_directory,
-            environment=environment.make_environment(config),
+            extra_environment=borg_environment,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -430,7 +563,7 @@ def create_archive(
         return execute_command_and_capture_output(
             create_flags + create_positional_arguments,
             working_directory=working_directory,
-            environment=environment.make_environment(config),
+            extra_environment=borg_environment,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -440,7 +573,7 @@ def create_archive(
             output_log_level,
             output_file,
             working_directory=working_directory,
-            environment=environment.make_environment(config),
+            extra_environment=borg_environment,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/delete.py

@@ -1,135 +0,0 @@
-import argparse
-import logging
-
-import borgmatic.borg.environment
-import borgmatic.borg.feature
-import borgmatic.borg.flags
-import borgmatic.borg.repo_delete
-import borgmatic.config.paths
-import borgmatic.execute
-
-logger = logging.getLogger(__name__)
-
-
-def make_delete_command(
-    repository,
-    config,
-    local_borg_version,
-    delete_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Given a local or remote repository dict, a configuration dict, the local Borg version, the
-    arguments to the delete action as an argparse.Namespace, and global arguments, return a command
-    as a tuple to delete archives from the repository.
-    '''
-    return (
-        (local_path, 'delete')
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + borgmatic.borg.flags.make_flags('dry-run', global_arguments.dry_run)
-        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
-        + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
-        + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
-        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
-        + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives)
-        + (
-            (('--force',) + (('--force',) if delete_arguments.force >= 2 else ()))
-            if delete_arguments.force
-            else ()
-        )
-        # Ignore match_archives and archive_name_format options from configuration, so the user has
-        # to be explicit on the command-line about the archives they want to delete.
-        + borgmatic.borg.flags.make_match_archives_flags(
-            delete_arguments.match_archives or delete_arguments.archive,
-            archive_name_format=None,
-            local_borg_version=local_borg_version,
-            default_archive_name_format='*',
-        )
-        + borgmatic.borg.flags.make_flags_from_arguments(
-            delete_arguments,
-            excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'),
-        )
-        + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
-    )
-
-
-ARCHIVE_RELATED_ARGUMENT_NAMES = (
-    'archive',
-    'match_archives',
-    'first',
-    'last',
-    'oldest',
-    'newest',
-    'older',
-    'newer',
-)
-
-
-def delete_archives(
-    repository,
-    config,
-    local_borg_version,
-    delete_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository dict, a configuration dict, the local Borg version, the
-    arguments to the delete action as an argparse.Namespace, global arguments as an
-    argparse.Namespace, and local and remote Borg paths, delete the selected archives from the
-    repository. If no archives are selected, then delete the entire repository.
-    '''
-    borgmatic.logger.add_custom_log_levels()
-
-    if not any(
-        getattr(delete_arguments, argument_name, None)
-        for argument_name in ARCHIVE_RELATED_ARGUMENT_NAMES
-    ):
-        if borgmatic.borg.feature.available(
-            borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
-        ):
-            logger.warning(
-                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.'
-            )
-
-        repo_delete_arguments = argparse.Namespace(
-            repository=repository['path'],
-            list_archives=delete_arguments.list_archives,
-            force=delete_arguments.force,
-            cache_only=delete_arguments.cache_only,
-            keep_security_info=delete_arguments.keep_security_info,
-        )
-        borgmatic.borg.repo_delete.delete_repository(
-            repository,
-            config,
-            local_borg_version,
-            repo_delete_arguments,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-
-        return
-
-    command = make_delete_command(
-        repository,
-        config,
-        local_borg_version,
-        delete_arguments,
-        global_arguments,
-        local_path,
-        remote_path,
-    )
-
-    borgmatic.execute.execute_command(
-        command,
-        output_log_level=logging.ANSWER,
-        environment=borgmatic.borg.environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )

borgmatic/borg/environment.py

@@ -1,8 +1,3 @@
-import os
-
-import borgmatic.borg.passcommand
-import borgmatic.hooks.credential.parse
-
 OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_base_directory': 'BORG_BASE_DIR',
     'borg_config_directory': 'BORG_CONFIG_DIR',
@@ -10,6 +5,8 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
     'borg_security_directory': 'BORG_SECURITY_DIR',
     'borg_keys_directory': 'BORG_KEYS_DIR',
+    'encryption_passcommand': 'BORG_PASSCOMMAND',
+    'encryption_passphrase': 'BORG_PASSPHRASE',
     'ssh_command': 'BORG_RSH',
     'temporary_directory': 'TMPDIR',
 }
@@ -26,65 +23,23 @@ DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = {
 
 def make_environment(config):
     '''
-    Given a borgmatic configuration dict, convert it to a Borg environment variable dict, merge it
-    with a copy of the current environment variables, and return the result.
-
-    Do not reuse this environment across multiple Borg invocations, because it can include
-    references to resources like anonymous pipes for passphrases—which can only be consumed once.
-
-    Here's how native Borg precedence works for a few of the environment variables:
-
-      1. BORG_PASSPHRASE, if set, is used first.
-      2. BORG_PASSCOMMAND is used only if BORG_PASSPHRASE isn't set.
-      3. BORG_PASSPHRASE_FD is used only if neither of the above are set.
-
-    In borgmatic, we want to simulate this precedence order, but there are some additional
-    complications. First, values can come from either configuration or from environment variables
-    set outside borgmatic; configured options should take precedence. Second, when borgmatic gets a
-    passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we
-    want to pass that passphrase to Borg via an anonymous pipe (+ BORG_PASSPHRASE_FD), since that's
-    more secure than using an environment variable (BORG_PASSPHRASE).
+    Given a borgmatic configuration dict, return its options converted to a Borg environment
+    variable dict.
     '''
-    environment = dict(os.environ)
+    environment = {}
 
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
         value = config.get(option_name)
 
-        if value is not None:
+        if value:
             environment[environment_variable_name] = str(value)
 
-    if 'encryption_passphrase' in config:
-        environment.pop('BORG_PASSPHRASE', None)
-        environment.pop('BORG_PASSCOMMAND', None)
-
-    if 'encryption_passcommand' in config:
-        environment.pop('BORG_PASSCOMMAND', None)
-
-    passphrase = borgmatic.hooks.credential.parse.resolve_credential(
-        config.get('encryption_passphrase'), config
-    )
-
-    if passphrase is None:
-        passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config)
-
-    # If there's a passphrase (from configuration, from a configured credential, or from a
-    # configured passcommand), send it to Borg via an anonymous pipe.
-    if passphrase is not None:
-        read_file_descriptor, write_file_descriptor = os.pipe()
-        os.write(write_file_descriptor, passphrase.encode('utf-8'))
-        os.close(write_file_descriptor)
-
-        # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the Borg
-        # child process to inherit the file descriptor.
-        os.set_inheritable(read_file_descriptor, True)
-        environment['BORG_PASSPHRASE_FD'] = str(read_file_descriptor)
-
     for (
         option_name,
         environment_variable_name,
     ) in DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE.items():
-        if os.environ.get(environment_variable_name) is None:
-            value = config.get(option_name)
+        value = config.get(option_name)
+        if value is not None:
             environment[environment_variable_name] = 'yes' if value else 'no'
 
     for (

borgmatic/borg/export_key.py

@@ -1,7 +1,6 @@
 import logging
 import os
 
-import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@@ -19,9 +18,9 @@ def export_key(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, export
-    arguments, and optional local and remote Borg paths, export the repository key to the
-    destination path indicated in the export arguments.
+    Given a local or remote repository path, a configuration dict, the local Borg version, and
+    optional local and remote Borg paths, export the repository key to the destination path
+    indicated in the export arguments.
 
     If the destination path is empty or "-", then print the key to stdout instead of to a file.
 
@@ -30,10 +29,9 @@ def export_key(
     borgmatic.logger.add_custom_log_levels()
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
-    working_directory = borgmatic.config.paths.get_working_directory(config)
 
     if export_arguments.path and export_arguments.path != '-':
-        if os.path.exists(os.path.join(working_directory or '', export_arguments.path)):
+        if os.path.exists(export_arguments.path):
             raise FileExistsError(
                 f'Destination path {export_arguments.path} already exists. Aborting.'
             )
@@ -60,15 +58,14 @@ def export_key(
     )
 
     if global_arguments.dry_run:
-        logger.info('Skipping key export (dry run)')
+        logging.info(f'{repository_path}: Skipping key export (dry run)')
         return
 
     execute_command(
         full_command,
         output_file=output_file,
         output_log_level=logging.ANSWER,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/export_tar.py

@@ -1,6 +1,5 @@
 import logging
 
-import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@@ -63,15 +62,14 @@ def export_tar_archive(
         output_log_level = logging.INFO
 
     if dry_run:
-        logging.info('Skipping export to tar file (dry run)')
+        logging.info(f'{repository_path}: Skipping export to tar file (dry run)')
         return
 
     execute_command(
         full_command,
         output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
         output_log_level=output_log_level,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/extract.py

@@ -2,9 +2,8 @@ import logging
 import os
 import subprocess
 
-import borgmatic.config.paths
 import borgmatic.config.validate
-from borgmatic.borg import environment, feature, flags, repo_list
+from borgmatic.borg import environment, feature, flags, rlist
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
@@ -30,7 +29,7 @@ def extract_last_archive_dry_run(
         verbosity_flags = ('--info',)
 
     try:
-        last_archive_name = repo_list.resolve_archive_name(
+        last_archive_name = rlist.resolve_archive_name(
             repository_path,
             'latest',
             config,
@@ -44,6 +43,7 @@ def extract_last_archive_dry_run(
         return
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
+    borg_environment = environment.make_environment(config)
     full_extract_command = (
         (local_path, 'extract', '--dry-run')
         + (('--remote-path', remote_path) if remote_path else ())
@@ -58,8 +58,8 @@ def extract_last_archive_dry_run(
 
     execute_command(
         full_extract_command,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        working_directory=None,
+        extra_environment=borg_environment,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
@@ -104,18 +104,8 @@ def extract_archive(
         if not paths:
             raise ValueError('The --strip-components flag with "all" requires at least one --path')
 
-        # Calculate the maximum number of leading path components of the given paths. "if piece"
-        # ignores empty path components, e.g. those resulting from a leading slash. And the "- 1"
-        # is so this doesn't count the final path component, e.g. the filename itself.
-        strip_components = max(
-            0,
-            *(
-                len(tuple(piece for piece in path.split(os.path.sep) if piece)) - 1
-                for path in paths
-            ),
-        )
-
-    working_directory = borgmatic.config.paths.get_working_directory(config)
+        # Calculate the maximum number of leading path components of the given paths.
+        strip_components = max(0, *(len(path.split(os.path.sep)) - 1 for path in paths))
 
     full_command = (
         (local_path, 'extract')
@@ -131,20 +121,17 @@ def extract_archive(
         + (('--progress',) if progress else ())
         + (('--stdout',) if extract_to_stdout else ())
         + flags.make_repository_archive_flags(
-            # Make the repository path absolute so the destination directory used below via changing
-            # the working directory doesn't prevent Borg from finding the repo. But also apply the
-            # user's configured working directory (if any) to the repo path.
-            borgmatic.config.validate.normalize_repository_path(repository, working_directory),
+            # Make the repository path absolute so the working directory changes below don't
+            # prevent Borg from finding the repo.
+            borgmatic.config.validate.normalize_repository_path(repository),
             archive,
             local_borg_version,
         )
         + (tuple(paths) if paths else ())
     )
 
+    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
-    full_destination_path = (
-        os.path.join(working_directory or '', destination_path) if destination_path else None
-    )
 
     # The progress output isn't compatible with captured and logged output, as progress messes with
     # the terminal directly.
@@ -152,8 +139,8 @@ def extract_archive(
         return execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            environment=environment.make_environment(config),
-            working_directory=full_destination_path,
+            working_directory=destination_path,
+            extra_environment=borg_environment,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -163,9 +150,9 @@ def extract_archive(
         return execute_command(
             full_command,
             output_file=subprocess.PIPE,
+            working_directory=destination_path,
             run_to_completion=False,
-            environment=environment.make_environment(config),
-            working_directory=full_destination_path,
+            extra_environment=borg_environment,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -174,8 +161,8 @@ def extract_archive(
     # if the restore paths don't exist in the archive.
     execute_command(
         full_command,
-        environment=environment.make_environment(config),
-        working_directory=full_destination_path,
+        working_directory=destination_path,
+        extra_environment=borg_environment,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )

borgmatic/borg/feature.py

@@ -10,14 +10,11 @@ class Feature(Enum):
     NUMERIC_IDS = 4
     UPLOAD_RATELIMIT = 5
     SEPARATE_REPOSITORY_ARCHIVE = 6
-    REPO_CREATE = 7
-    REPO_LIST = 8
-    REPO_INFO = 9
-    REPO_DELETE = 10
-    MATCH_ARCHIVES = 11
-    EXCLUDED_FILES_MINUS = 12
-    ARCHIVE_SERIES = 13
-    NO_PRUNE_STATS = 14
+    RCREATE = 7
+    RLIST = 8
+    RINFO = 9
+    MATCH_ARCHIVES = 10
+    EXCLUDED_FILES_MINUS = 11
 
 
 FEATURE_TO_MINIMUM_BORG_VERSION = {
@@ -27,14 +24,11 @@ FEATURE_TO_MINIMUM_BORG_VERSION = {
     Feature.NUMERIC_IDS: parse('1.2.0b3'),  # borg create/extract/mount --numeric-ids
     Feature.UPLOAD_RATELIMIT: parse('1.2.0b3'),  # borg create --upload-ratelimit
     Feature.SEPARATE_REPOSITORY_ARCHIVE: parse('2.0.0a2'),  # --repo with separate archive
-    Feature.REPO_CREATE: parse('2.0.0a2'),  # borg repo-create
-    Feature.REPO_LIST: parse('2.0.0a2'),  # borg repo-list
-    Feature.REPO_INFO: parse('2.0.0a2'),  # borg repo-info
-    Feature.REPO_DELETE: parse('2.0.0a2'),  # borg repo-delete
+    Feature.RCREATE: parse('2.0.0a2'),  # borg rcreate
+    Feature.RLIST: parse('2.0.0a2'),  # borg rlist
+    Feature.RINFO: parse('2.0.0a2'),  # borg rinfo
     Feature.MATCH_ARCHIVES: parse('2.0.0b3'),  # borg --match-archives
     Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'),  # --list --filter uses "-" for excludes
-    Feature.ARCHIVE_SERIES: parse('2.0.0b11'),  # identically named archives form a series
-    Feature.NO_PRUNE_STATS: parse('2.0.0b10'),  # prune --stats is not available
 }
 
 

borgmatic/borg/flags.py

@@ -50,9 +50,6 @@ def make_repository_flags(repository_path, local_borg_version):
     ) + (repository_path,)
 
 
-ARCHIVE_HASH_PATTERN = re.compile('[0-9a-fA-F]{8,}$')
-
-
 def make_repository_archive_flags(repository_path, archive, local_borg_version):
     '''
     Given the path of a Borg repository, an archive name or pattern, and the local Borg version,
@@ -60,42 +57,16 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
     and archive.
     '''
     return (
-        (
-            '--repo',
-            repository_path,
-            (
-                f'aid:{archive}'
-                if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
-                and ARCHIVE_HASH_PATTERN.match(archive)
-                and not archive.startswith('aid:')
-                else archive
-            ),
-        )
+        ('--repo', repository_path, archive)
         if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version)
         else (f'{repository_path}::{archive}',)
     )
 
 
-DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
-DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'  # noqa: FS003
+DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
 
 
-def get_default_archive_name_format(local_borg_version):
-    '''
-    Given the local Borg version, return the corresponding default archive name format.
-    '''
-    if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version):
-        return DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES
-
-    return DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES
-
-
-def make_match_archives_flags(
-    match_archives,
-    archive_name_format,
-    local_borg_version,
-    default_archive_name_format=None,
-):
+def make_match_archives_flags(match_archives, archive_name_format, local_borg_version):
     '''
     Return match archives flags based on the given match archives value, if any. If it isn't set,
     return match archives flags to match archives created with the given (or default) archive name
@@ -107,23 +78,12 @@ def make_match_archives_flags(
             return ()
 
         if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
-            if (
-                feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
-                and ARCHIVE_HASH_PATTERN.match(match_archives)
-                and not match_archives.startswith('aid:')
-            ):
-                return ('--match-archives', f'aid:{match_archives}')
-
             return ('--match-archives', match_archives)
         else:
             return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
 
     derived_match_archives = re.sub(
-        r'\{(now|utcnow|pid)([:%\w\.-]*)\}',
-        '*',
-        archive_name_format
-        or default_archive_name_format
-        or get_default_archive_name_format(local_borg_version),
+        r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format or DEFAULT_ARCHIVE_NAME_FORMAT
     )
 
     if derived_match_archives == '*':
@@ -156,44 +116,3 @@ def warn_for_aggressive_archive_flags(json_command, json_output):
         logger.debug(f'Cannot parse JSON output from archive command: {error}')
     except (TypeError, KeyError):
         logger.debug('Cannot parse JSON output from archive command: No "archives" key found')
-
-
-def omit_flag(arguments, flag):
-    '''
-    Given a sequence of Borg command-line arguments, return them with the given (valueless) flag
-    omitted. For instance, if the flag is "--flag" and arguments is:
-
-        ('borg', 'create', '--flag', '--other-flag')
-
-    ... then return:
-
-        ('borg', 'create', '--other-flag')
-    '''
-    return tuple(argument for argument in arguments if argument != flag)
-
-
-def omit_flag_and_value(arguments, flag):
-    '''
-    Given a sequence of Borg command-line arguments, return them with the given flag and its
-    corresponding value omitted. For instance, if the flag is "--flag" and arguments is:
-
-        ('borg', 'create', '--flag', 'value', '--other-flag')
-
-    ... or:
-
-        ('borg', 'create', '--flag=value', '--other-flag')
-
-    ... then return:
-
-        ('borg', 'create', '--other-flag')
-    '''
-    # This works by zipping together a list of overlapping pairwise arguments. E.g., ('one', 'two',
-    # 'three', 'four') becomes ((None, 'one'), ('one, 'two'), ('two', 'three'), ('three', 'four')).
-    # This makes it easy to "look back" at the previous arguments so we can exclude both a flag and
-    # its value.
-    return tuple(
-        argument
-        for (previous_argument, argument) in zip((None,) + arguments, arguments)
-        if flag not in (previous_argument, argument)
-        if not argument.startswith(f'{flag}=')
-    )

borgmatic/borg/import_key.py

@@ -1,70 +0,0 @@
-import logging
-import os
-
-import borgmatic.config.paths
-import borgmatic.logger
-from borgmatic.borg import environment, flags
-from borgmatic.execute import DO_NOT_CAPTURE, execute_command
-
-logger = logging.getLogger(__name__)
-
-
-def import_key(
-    repository_path,
-    config,
-    local_borg_version,
-    import_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, import
-    arguments, and optional local and remote Borg paths, import the repository key from the
-    path indicated in the import arguments.
-
-    If the path is empty or "-", then read the key from stdin.
-
-    Raise ValueError if the path is given and it does not exist.
-    '''
-    umask = config.get('umask', None)
-    lock_wait = config.get('lock_wait', None)
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    if import_arguments.path and import_arguments.path != '-':
-        if not os.path.exists(os.path.join(working_directory or '', import_arguments.path)):
-            raise ValueError(f'Path {import_arguments.path} does not exist. Aborting.')
-
-        input_file = None
-    else:
-        input_file = DO_NOT_CAPTURE
-
-    full_command = (
-        (local_path, 'key', 'import')
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + flags.make_flags('paper', import_arguments.paper)
-        + flags.make_repository_flags(
-            repository_path,
-            local_borg_version,
-        )
-        + ((import_arguments.path,) if input_file is None else ())
-    )
-
-    if global_arguments.dry_run:
-        logger.info('Skipping key import (dry run)')
-        return
-
-    execute_command(
-        full_command,
-        input_file=input_file,
-        output_log_level=logging.INFO,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )

borgmatic/borg/info.py

@@ -1,7 +1,6 @@
 import argparse
 import logging
 
-import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import execute_command, execute_command_and_capture_output
@@ -36,7 +35,6 @@ def make_info_command(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', config.get('lock_wait'))
         + (
@@ -98,12 +96,10 @@ def display_archives_info(
         remote_path,
     )
     borg_exit_codes = config.get('borg_exit_codes')
-    working_directory = borgmatic.config.paths.get_working_directory(config)
 
     json_info = execute_command_and_capture_output(
         json_command,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )
@@ -116,8 +112,7 @@ def display_archives_info(
     execute_command(
         main_command,
         output_log_level=logging.ANSWER,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )

borgmatic/borg/list.py

@@ -3,21 +3,20 @@ import copy
 import logging
 import re
 
-import borgmatic.config.paths
 import borgmatic.logger
-from borgmatic.borg import environment, feature, flags, repo_list
+from borgmatic.borg import environment, feature, flags, rlist
 from borgmatic.execute import execute_command, execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
 
-ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST = ('prefix', 'match_archives', 'sort_by', 'first', 'last')
+ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST = ('prefix', 'match_archives', 'sort_by', 'first', 'last')
 MAKE_FLAGS_EXCLUDES = (
     'repository',
     'archive',
     'paths',
     'find_paths',
-) + ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST
+) + ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST
 
 
 def make_list_command(
@@ -34,6 +33,8 @@ def make_list_command(
     and local and remote Borg paths, return a command as a tuple to list archives or paths within an
     archive.
     '''
+    lock_wait = config.get('lock_wait', None)
+
     return (
         (local_path, 'list')
         + (
@@ -47,9 +48,8 @@ def make_list_command(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
-        + flags.make_flags('lock-wait', config.get('lock_wait'))
+        + flags.make_flags('lock-wait', lock_wait)
         + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + (
             flags.make_repository_archive_flags(
@@ -100,12 +100,13 @@ def capture_archive_listing(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, an archive name, a configuration
-    dict, the local Borg version, global arguments as an argparse.Namespace,
-    the archive paths (or Borg patterns) in which to list files, the Borg path
-    format to use for the output, and local and remote Borg paths, capture the
-    output of listing that archive and return it as a list of file paths.
+    Given a local or remote repository path, an archive name, a configuration dict, the local Borg
+    version, global arguments as an argparse.Namespace, the archive paths in which to list files,
+    the Borg path format to use for the output, and local and remote Borg paths, capture the output
+    of listing that archive and return it as a list of file paths.
     '''
+    borg_environment = environment.make_environment(config)
+
     return tuple(
         execute_command_and_capture_output(
             make_list_command(
@@ -115,22 +116,21 @@ def capture_archive_listing(
                 argparse.Namespace(
                     repository=repository_path,
                     archive=archive,
-                    paths=[path for path in list_paths] if list_paths else None,
+                    paths=[f'sh:{path}' for path in list_paths] if list_paths else None,
                     find_paths=None,
                     json=None,
-                    format=path_format or '{path}{NUL}',  # noqa: FS003
+                    format=path_format or '{path}{NL}',  # noqa: FS003
                 ),
                 global_arguments,
                 local_path,
                 remote_path,
             ),
-            environment=environment.make_environment(config),
-            working_directory=borgmatic.config.paths.get_working_directory(config),
+            extra_environment=borg_environment,
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
         )
-        .strip('\0')
-        .split('\0')
+        .strip('\n')
+        .split('\n')
     )
 
 
@@ -144,22 +144,22 @@ def list_archive(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, the
-    arguments to the list action as an argparse.Namespace, global arguments as an
-    argparse.Namespace, and local and remote Borg paths, display the output of listing the files of
-    a Borg archive (or return JSON output). If list_arguments.find_paths are given, list the files
-    by searching across multiple archives. If neither find_paths nor archive name are given, instead
-    list the archives in the given repository.
+    Given a local or remote repository path, a configuration dict, the local Borg version, global
+    arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace,
+    and local and remote Borg paths, display the output of listing the files of a Borg archive (or
+    return JSON output). If list_arguments.find_paths are given, list the files by searching across
+    multiple archives. If neither find_paths nor archive name are given, instead list the archives
+    in the given repository.
     '''
     borgmatic.logger.add_custom_log_levels()
 
     if not list_arguments.archive and not list_arguments.find_paths:
-        if feature.available(feature.Feature.REPO_LIST, local_borg_version):
+        if feature.available(feature.Feature.RLIST, local_borg_version):
             logger.warning(
-                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.'
+                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the rlist action instead.'
             )
 
-        repo_list_arguments = argparse.Namespace(
+        rlist_arguments = argparse.Namespace(
             repository=repository_path,
             short=list_arguments.short,
             format=list_arguments.format,
@@ -170,18 +170,18 @@ def list_archive(
             first=list_arguments.first,
             last=list_arguments.last,
         )
-        return repo_list.list_repository(
+        return rlist.list_repository(
             repository_path,
             config,
             local_borg_version,
-            repo_list_arguments,
+            rlist_arguments,
             global_arguments,
             local_path,
             remote_path,
         )
 
     if list_arguments.archive:
-        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
+        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST:
             if getattr(list_arguments, name, None):
                 logger.warning(
                     f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag."
@@ -192,12 +192,13 @@ def list_archive(
             'The --json flag on the list action is not supported when using the --archive/--find flags.'
         )
 
+    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
     # If there are any paths to find (and there's not a single archive already selected), start by
     # getting a list of archives to search.
     if list_arguments.find_paths and not list_arguments.archive:
-        repo_list_arguments = argparse.Namespace(
+        rlist_arguments = argparse.Namespace(
             repository=repository_path,
             short=True,
             format=None,
@@ -212,36 +213,35 @@ def list_archive(
         # Ask Borg to list archives. Capture its output for use below.
         archive_lines = tuple(
             execute_command_and_capture_output(
-                repo_list.make_repo_list_command(
+                rlist.make_rlist_command(
                     repository_path,
                     config,
                     local_borg_version,
-                    repo_list_arguments,
+                    rlist_arguments,
                     global_arguments,
                     local_path,
                     remote_path,
                 ),
-                environment=environment.make_environment(config),
-                working_directory=borgmatic.config.paths.get_working_directory(config),
+                extra_environment=borg_environment,
                 borg_local_path=local_path,
                 borg_exit_codes=borg_exit_codes,
             )
             .strip('\n')
-            .splitlines()
+            .split('\n')
         )
     else:
         archive_lines = (list_arguments.archive,)
 
     # For each archive listed by Borg, run list on the contents of that archive.
     for archive in archive_lines:
-        logger.answer(f'Listing archive {archive}')
+        logger.answer(f'{repository_path}: Listing archive {archive}')
 
         archive_arguments = copy.copy(list_arguments)
         archive_arguments.archive = archive
 
         # This list call is to show the files in a single archive, not list multiple archives. So
         # blank out any archive filtering flags. They'll break anyway in Borg 2.
-        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
+        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST:
             setattr(archive_arguments, name, None)
 
         main_command = make_list_command(
@@ -257,8 +257,7 @@ def list_archive(
         execute_command(
             main_command,
             output_log_level=logging.ANSWER,
-            environment=environment.make_environment(config),
-            working_directory=borgmatic.config.paths.get_working_directory(config),
+            extra_environment=borg_environment,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/mount.py

@@ -1,6 +1,5 @@
 import logging
 
-import borgmatic.config.paths
 from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
@@ -59,15 +58,14 @@ def mount_archive(
         + (tuple(mount_arguments.paths) if mount_arguments.paths else ())
     )
 
-    working_directory = borgmatic.config.paths.get_working_directory(config)
+    borg_environment = environment.make_environment(config)
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
     if mount_arguments.foreground:
         execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            environment=environment.make_environment(config),
-            working_directory=working_directory,
+            extra_environment=borg_environment,
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
         )
@@ -75,8 +73,7 @@ def mount_archive(
 
     execute_command(
         full_command,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
+        extra_environment=borg_environment,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/passcommand.py

@@ -1,40 +0,0 @@
-import functools
-import logging
-import shlex
-
-import borgmatic.config.paths
-import borgmatic.execute
-
-logger = logging.getLogger(__name__)
-
-
-@functools.cache
-def run_passcommand(passcommand, working_directory):
-    '''
-    Run the given passcommand using the given working directory and return the passphrase produced
-    by the command.
-
-    Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
-    per borgmatic invocation.
-    '''
-    return borgmatic.execute.execute_command_and_capture_output(
-        shlex.split(passcommand),
-        working_directory=working_directory,
-    )
-
-
-def get_passphrase_from_passcommand(config):
-    '''
-    Given the configuration dict, call the configured passcommand to produce and return an
-    encryption passphrase. In effect, we're doing an end-run around Borg by invoking its passcommand
-    ourselves. This allows us to pass the resulting passphrase to multiple different Borg
-    invocations without the user having to be prompted multiple times.
-
-    If no passcommand is configured, then return None.
-    '''
-    passcommand = config.get('encryption_passcommand')
-
-    if not passcommand:
-        return None
-
-    return run_passcommand(passcommand, borgmatic.config.paths.get_working_directory(config))

borgmatic/borg/pattern.py

@@ -1,50 +0,0 @@
-import collections
-import enum
-
-
-# See https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns
-class Pattern_type(enum.Enum):
-    ROOT = 'R'  # A ROOT pattern always has a NONE pattern style.
-    PATTERN_STYLE = 'P'
-    EXCLUDE = '-'
-    NO_RECURSE = '!'
-    INCLUDE = '+'
-
-
-class Pattern_style(enum.Enum):
-    NONE = ''
-    FNMATCH = 'fm'
-    SHELL = 'sh'
-    REGULAR_EXPRESSION = 're'
-    PATH_PREFIX = 'pp'
-    PATH_FULL_MATCH = 'pf'
-
-
-class Pattern_source(enum.Enum):
-    '''
-    Where the pattern came from within borgmatic. This is important because certain use cases (like
-    filesystem snapshotting) only want to consider patterns that the user actually put in a
-    configuration file and not patterns from other sources.
-    '''
-
-    # The pattern is from a borgmatic configuration option, e.g. listed in "source_directories".
-    CONFIG = 'config'
-
-    # The pattern is generated internally within borgmatic, e.g. for special file excludes.
-    INTERNAL = 'internal'
-
-    # The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump
-    # directory.
-    HOOK = 'hook'
-
-
-Pattern = collections.namedtuple(
-    'Pattern',
-    ('path', 'type', 'style', 'device', 'source'),
-    defaults=(
-        Pattern_type.ROOT,
-        Pattern_style.NONE,
-        None,
-        Pattern_source.HOOK,
-    ),
-)

borgmatic/borg/prune.py

@@ -1,6 +1,5 @@
 import logging
 
-import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import execute_command
@@ -8,10 +7,9 @@ from borgmatic.execute import execute_command
 logger = logging.getLogger(__name__)
 
 
-def make_prune_flags(config, prune_arguments, local_borg_version):
+def make_prune_flags(config, local_borg_version):
     '''
-    Given a configuration dict mapping from option name to value, prune arguments as an
-    argparse.Namespace instance, and the local Borg version, produce a corresponding sequence of
+    Given a configuration dict mapping from option name to value, transform it into an sequence of
     command-line flags.
 
     For example, given a retention config of:
@@ -41,7 +39,7 @@ def make_prune_flags(config, prune_arguments, local_borg_version):
         if prefix
         else (
             flags.make_match_archives_flags(
-                prune_arguments.match_archives or config.get('match_archives'),
+                config.get('match_archives'),
                 config.get('archive_name_format'),
                 local_borg_version,
             )
@@ -70,22 +68,16 @@ def prune_archives(
 
     full_command = (
         (local_path, 'prune')
-        + make_prune_flags(config, prune_arguments, local_borg_version)
+        + make_prune_flags(config, local_borg_version)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (
-            ('--stats',)
-            if prune_arguments.stats
-            and not dry_run
-            and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version)
-            else ()
-        )
+        + (('--stats',) if prune_arguments.stats and not dry_run else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + flags.make_flags_from_arguments(
             prune_arguments,
-            excludes=('repository', 'match_archives', 'stats', 'list_archives'),
+            excludes=('repository', 'stats', 'list_archives'),
         )
         + (('--list',) if prune_arguments.list_archives else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
@@ -102,8 +94,7 @@ def prune_archives(
     execute_command(
         full_command,
         output_log_level=output_log_level,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/rcreate.py

@@ -3,14 +3,13 @@ import json
 import logging
 import subprocess
 
-import borgmatic.config.paths
-from borgmatic.borg import environment, feature, flags, repo_info
+from borgmatic.borg import environment, feature, flags, rinfo
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
 
-REPO_INFO_REPOSITORY_NOT_FOUND_EXIT_CODES = {2, 13, 15}
+RINFO_REPOSITORY_NOT_FOUND_EXIT_CODES = {2, 13}
 
 
 def create_repository(
@@ -40,7 +39,7 @@ def create_repository(
     '''
     try:
         info_data = json.loads(
-            repo_info.display_repository_info(
+            rinfo.display_repository_info(
                 repository_path,
                 config,
                 local_borg_version,
@@ -57,21 +56,20 @@ def create_repository(
                 f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"'
             )
 
-        logger.info('Repository already exists. Skipping creation.')
+        logger.info(f'{repository_path}: Repository already exists. Skipping creation.')
         return
     except subprocess.CalledProcessError as error:
-        if error.returncode not in REPO_INFO_REPOSITORY_NOT_FOUND_EXIT_CODES:
+        if error.returncode not in RINFO_REPOSITORY_NOT_FOUND_EXIT_CODES:
             raise
 
     lock_wait = config.get('lock_wait')
-    umask = config.get('umask')
-    extra_borg_options = config.get('extra_borg_options', {}).get('repo-create', '')
+    extra_borg_options = config.get('extra_borg_options', {}).get('rcreate', '')
 
-    repo_create_command = (
+    rcreate_command = (
         (local_path,)
         + (
-            ('repo-create',)
-            if feature.available(feature.Feature.REPO_CREATE, local_borg_version)
+            ('rcreate',)
+            if feature.available(feature.Feature.RCREATE, local_borg_version)
             else ('init',)
         )
         + (('--encryption', encryption_mode) if encryption_mode else ())
@@ -85,21 +83,19 @@ def create_repository(
         + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     if dry_run:
-        logging.info('Skipping repository creation (dry run)')
+        logging.info(f'{repository_path}: Skipping repository creation (dry run)')
         return
 
     # Do not capture output here, so as to support interactive prompts.
     execute_command(
-        repo_create_command,
+        rcreate_command,
         output_file=DO_NOT_CAPTURE,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/recreate.py

@@ -1,100 +0,0 @@
-import logging
-import shlex
-
-import borgmatic.borg.environment
-import borgmatic.config.paths
-import borgmatic.execute
-from borgmatic.borg import flags
-from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags, write_patterns_file
-
-logger = logging.getLogger(__name__)
-
-
-def recreate_archive(
-    repository,
-    archive,
-    config,
-    local_borg_version,
-    recreate_arguments,
-    global_arguments,
-    local_path,
-    remote_path=None,
-    patterns=None,
-):
-    '''
-    Given a local or remote repository path, an archive name, a configuration dict,
-    the local Borg version string, an argparse.Namespace of recreate arguments,
-    an argparse.Namespace of global arguments, optional local and remote Borg paths.
-
-    Executes the recreate command with the given arguments.
-    '''
-
-    lock_wait = config.get('lock_wait', None)
-    exclude_flags = make_exclude_flags(config)
-    compression = config.get('compression', None)
-    chunker_params = config.get('chunker_params', None)
-    # Available recompress MODES: 'if-different' (default), 'always', 'never'
-    recompress = config.get('recompress', None)
-
-    # Write patterns to a temporary file and use that file with --patterns-from.
-    patterns_file = write_patterns_file(
-        patterns, borgmatic.config.paths.get_working_directory(config)
-    )
-
-    recreate_command = (
-        (local_path, 'recreate')
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait is not None else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + (('--patterns-from', patterns_file.name) if patterns_file else ())
-        + (
-            (
-                '--list',
-                '--filter',
-                make_list_filter_flags(local_borg_version, global_arguments.dry_run),
-            )
-            if recreate_arguments.list
-            else ()
-        )
-        # Flag --target works only for a single archive
-        + (('--target', recreate_arguments.target) if recreate_arguments.target and archive else ())
-        + (
-            ('--comment', shlex.quote(recreate_arguments.comment))
-            if recreate_arguments.comment
-            else ()
-        )
-        + (('--timestamp', recreate_arguments.timestamp) if recreate_arguments.timestamp else ())
-        + (('--compression', compression) if compression else ())
-        + (('--chunker-params', chunker_params) if chunker_params else ())
-        + (
-            flags.make_match_archives_flags(
-                recreate_arguments.match_archives or archive or config.get('match_archives'),
-                config.get('archive_name_format'),
-                local_borg_version,
-            )
-            if recreate_arguments.match_archives
-            else ()
-        )
-        + (('--recompress', recompress) if recompress else ())
-        + exclude_flags
-        + (
-            flags.make_repository_archive_flags(repository, archive, local_borg_version)
-            if archive
-            else flags.make_repository_flags(repository, local_borg_version)
-        )
-    )
-
-    if global_arguments.dry_run:
-        logger.info('Skipping the archive recreation (dry run)')
-        return
-
-    borgmatic.execute.execute_command(
-        full_command=recreate_command,
-        output_log_level=logging.INFO,
-        environment=borgmatic.borg.environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )

borgmatic/borg/repo_delete.py

@@ -1,95 +0,0 @@
-import logging
-
-import borgmatic.borg.environment
-import borgmatic.borg.feature
-import borgmatic.borg.flags
-import borgmatic.config.paths
-import borgmatic.execute
-
-logger = logging.getLogger(__name__)
-
-
-def make_repo_delete_command(
-    repository,
-    config,
-    local_borg_version,
-    repo_delete_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Given a local or remote repository dict, a configuration dict, the local Borg version, the
-    arguments to the repo_delete action as an argparse.Namespace, and global arguments, return a command
-    as a tuple to repo_delete the entire repository.
-    '''
-    return (
-        (local_path,)
-        + (
-            ('repo-delete',)
-            if borgmatic.borg.feature.available(
-                borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
-            )
-            else ('delete',)
-        )
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + borgmatic.borg.flags.make_flags('dry-run', global_arguments.dry_run)
-        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
-        + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
-        + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
-        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
-        + borgmatic.borg.flags.make_flags('list', repo_delete_arguments.list_archives)
-        + (
-            (('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ()))
-            if repo_delete_arguments.force
-            else ()
-        )
-        + borgmatic.borg.flags.make_flags_from_arguments(
-            repo_delete_arguments, excludes=('list_archives', 'force', 'repository')
-        )
-        + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
-    )
-
-
-def delete_repository(
-    repository,
-    config,
-    local_borg_version,
-    repo_delete_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository dict, a configuration dict, the local Borg version, the
-    arguments to the repo_delete action as an argparse.Namespace, global arguments as an
-    argparse.Namespace, and local and remote Borg paths, repo_delete the entire repository.
-    '''
-    borgmatic.logger.add_custom_log_levels()
-
-    command = make_repo_delete_command(
-        repository,
-        config,
-        local_borg_version,
-        repo_delete_arguments,
-        global_arguments,
-        local_path,
-        remote_path,
-    )
-
-    borgmatic.execute.execute_command(
-        command,
-        output_log_level=logging.ANSWER,
-        # Don't capture output when Borg is expected to prompt for interactive confirmation, or the
-        # prompt won't work.
-        output_file=(
-            None
-            if repo_delete_arguments.force or repo_delete_arguments.cache_only
-            else borgmatic.execute.DO_NOT_CAPTURE
-        ),
-        environment=borgmatic.borg.environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
-    )

borgmatic/borg/rinfo.py

@@ -1,6 +1,5 @@
 import logging
 
-import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import execute_command, execute_command_and_capture_output
@@ -12,14 +11,14 @@ def display_repository_info(
     repository_path,
     config,
     local_borg_version,
-    repo_info_arguments,
+    rinfo_arguments,
     global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, a configuration dict, the local Borg version, the
-    arguments to the repo_info action, and global arguments as an argparse.Namespace, display summary
+    arguments to the rinfo action, and global arguments as an argparse.Namespace, display summary
     information for the Borg repository or return JSON summary information.
     '''
     borgmatic.logger.add_custom_log_levels()
@@ -28,36 +27,34 @@ def display_repository_info(
     full_command = (
         (local_path,)
         + (
-            ('repo-info',)
-            if feature.available(feature.Feature.REPO_INFO, local_borg_version)
+            ('rinfo',)
+            if feature.available(feature.Feature.RINFO, local_borg_version)
             else ('info',)
         )
         + (
             ('--info',)
-            if logger.getEffectiveLevel() == logging.INFO and not repo_info_arguments.json
+            if logger.getEffectiveLevel() == logging.INFO and not rinfo_arguments.json
             else ()
         )
         + (
             ('--debug', '--show-rc')
-            if logger.isEnabledFor(logging.DEBUG) and not repo_info_arguments.json
+            if logger.isEnabledFor(logging.DEBUG) and not rinfo_arguments.json
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', lock_wait)
-        + (('--json',) if repo_info_arguments.json else ())
+        + (('--json',) if rinfo_arguments.json else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    working_directory = borgmatic.config.paths.get_working_directory(config)
+    extra_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
-    if repo_info_arguments.json:
+    if rinfo_arguments.json:
         return execute_command_and_capture_output(
             full_command,
-            environment=environment.make_environment(config),
-            working_directory=working_directory,
+            extra_environment=extra_environment,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -65,8 +62,7 @@ def display_repository_info(
         execute_command(
             full_command,
             output_log_level=logging.ANSWER,
-            environment=environment.make_environment(config),
-            working_directory=working_directory,
+            extra_environment=extra_environment,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/rlist.py

@@ -1,7 +1,6 @@
 import argparse
 import logging
 
-import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import execute_command, execute_command_and_capture_output
@@ -32,14 +31,9 @@ def resolve_archive_name(
     full_command = (
         (
             local_path,
-            (
-                'repo-list'
-                if feature.available(feature.Feature.REPO_LIST, local_borg_version)
-                else 'list'
-            ),
+            'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list',
         )
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', config.get('lock_wait'))
         + flags.make_flags('last', 1)
@@ -49,8 +43,7 @@ def resolve_archive_name(
 
     output = execute_command_and_capture_output(
         full_command,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
@@ -59,7 +52,7 @@ def resolve_archive_name(
     except IndexError:
         raise ValueError('No archives found in the repository')
 
-    logger.debug(f'Latest archive is {latest_archive}')
+    logger.debug(f'{repository_path}: Latest archive is {latest_archive}')
 
     return latest_archive
 
@@ -67,59 +60,54 @@ def resolve_archive_name(
 MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
 
 
-def make_repo_list_command(
+def make_rlist_command(
     repository_path,
     config,
     local_borg_version,
-    repo_list_arguments,
+    rlist_arguments,
     global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, a configuration dict, the local Borg version, the
-    arguments to the repo_list action, global arguments as an argparse.Namespace instance, and local and
+    arguments to the rlist action, global arguments as an argparse.Namespace instance, and local and
     remote Borg paths, return a command as a tuple to list archives with a repository.
     '''
     return (
         (
             local_path,
-            (
-                'repo-list'
-                if feature.available(feature.Feature.REPO_LIST, local_borg_version)
-                else 'list'
-            ),
+            'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list',
         )
         + (
             ('--info',)
-            if logger.getEffectiveLevel() == logging.INFO and not repo_list_arguments.json
+            if logger.getEffectiveLevel() == logging.INFO and not rlist_arguments.json
             else ()
         )
         + (
             ('--debug', '--show-rc')
-            if logger.isEnabledFor(logging.DEBUG) and not repo_list_arguments.json
+            if logger.isEnabledFor(logging.DEBUG) and not rlist_arguments.json
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', config.get('lock_wait'))
         + (
             (
-                flags.make_flags('match-archives', f'sh:{repo_list_arguments.prefix}*')
+                flags.make_flags('match-archives', f'sh:{rlist_arguments.prefix}*')
                 if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
-                else flags.make_flags('glob-archives', f'{repo_list_arguments.prefix}*')
+                else flags.make_flags('glob-archives', f'{rlist_arguments.prefix}*')
             )
-            if repo_list_arguments.prefix
+            if rlist_arguments.prefix
             else (
                 flags.make_match_archives_flags(
-                    repo_list_arguments.match_archives or config.get('match_archives'),
+                    rlist_arguments.match_archives or config.get('match_archives'),
                     config.get('archive_name_format'),
                     local_borg_version,
                 )
             )
         )
-        + flags.make_flags_from_arguments(repo_list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
+        + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
@@ -128,7 +116,7 @@ def list_repository(
     repository_path,
     config,
     local_borg_version,
-    repo_list_arguments,
+    rlist_arguments,
     global_arguments,
     local_path='borg',
     remote_path=None,
@@ -140,37 +128,36 @@ def list_repository(
     return JSON output).
     '''
     borgmatic.logger.add_custom_log_levels()
+    borg_environment = environment.make_environment(config)
 
-    main_command = make_repo_list_command(
+    main_command = make_rlist_command(
         repository_path,
         config,
         local_borg_version,
-        repo_list_arguments,
+        rlist_arguments,
         global_arguments,
         local_path,
         remote_path,
     )
-    json_command = make_repo_list_command(
+    json_command = make_rlist_command(
         repository_path,
         config,
         local_borg_version,
-        argparse.Namespace(**dict(repo_list_arguments.__dict__, json=True)),
+        argparse.Namespace(**dict(rlist_arguments.__dict__, json=True)),
         global_arguments,
         local_path,
         remote_path,
     )
-    working_directory = borgmatic.config.paths.get_working_directory(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
     json_listing = execute_command_and_capture_output(
         json_command,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
+        extra_environment=borg_environment,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )
 
-    if repo_list_arguments.json:
+    if rlist_arguments.json:
         return json_listing
 
     flags.warn_for_aggressive_archive_flags(json_command, json_listing)
@@ -178,8 +165,7 @@ def list_repository(
     execute_command(
         main_command,
         output_log_level=logging.ANSWER,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
+        extra_environment=borg_environment,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )

borgmatic/borg/transfer.py

@@ -1,6 +1,5 @@
 import logging
 
-import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@@ -30,7 +29,6 @@ def transfer_archives(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', config.get('lock_wait', None))
         + (
@@ -57,8 +55,7 @@ def transfer_archives(
         full_command,
         output_log_level=logging.ANSWER,
         output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
+        extra_environment=environment.make_environment(config),
     )

borgmatic/borg/umount.py

@@ -1,6 +1,5 @@
 import logging
 
-import borgmatic.config.paths
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
@@ -19,8 +18,5 @@ def unmount_archive(config, mount_point, local_path='borg'):
     )
 
     execute_command(
-        full_command,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
+        full_command, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes')
     )

borgmatic/borg/version.py

@@ -1,6 +1,5 @@
 import logging
 
-import borgmatic.config.paths
 from borgmatic.borg import environment
 from borgmatic.execute import execute_command_and_capture_output
 
@@ -9,7 +8,7 @@ logger = logging.getLogger(__name__)
 
 def local_borg_version(config, local_path='borg'):
     '''
-    Given a configuration dict and a local Borg executable path, return a version string for it.
+    Given a configuration dict and a local Borg binary path, return a version string for it.
 
     Raise OSError or CalledProcessError if there is a problem running Borg.
     Raise ValueError if the version cannot be parsed.
@@ -21,8 +20,7 @@ def local_borg_version(config, local_path='borg'):
     )
     output = execute_command_and_capture_output(
         full_command,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        extra_environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/commands/arguments.py

@@ -6,28 +6,25 @@ from argparse import ArgumentParser
 from borgmatic.config import collect
 
 ACTION_ALIASES = {
-    'repo-create': ['rcreate', 'init', '-I'],
+    'rcreate': ['init', '-I'],
     'prune': ['-p'],
     'compact': [],
     'create': ['-C'],
     'check': ['-k'],
     'config': [],
-    'delete': [],
     'extract': ['-x'],
     'export-tar': [],
     'mount': ['-m'],
     'umount': ['-u'],
     'restore': ['-r'],
-    'repo-delete': ['rdelete'],
-    'repo-list': ['rlist'],
+    'rlist': [],
     'list': ['-l'],
-    'repo-info': ['rinfo'],
+    'rinfo': [],
     'info': ['-i'],
     'transfer': [],
     'break-lock': [],
     'key': [],
     'borg': [],
-    'recreate': [],
 }
 
 
@@ -75,11 +72,11 @@ def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments
     for action_name, parsed in parsed_arguments.items():
         for value in vars(parsed).values():
             if isinstance(value, str):
-                if value in ACTION_ALIASES.keys() and value in remaining_arguments:
+                if value in ACTION_ALIASES.keys():
                     remaining_arguments.remove(value)
             elif isinstance(value, list):
                 for item in value:
-                    if item in ACTION_ALIASES.keys() and item in remaining_arguments:
+                    if item in ACTION_ALIASES.keys():
                         remaining_arguments.remove(item)
 
     return tuple(remaining_arguments)
@@ -116,54 +113,6 @@ def parse_and_record_action_arguments(
     return tuple(argument for argument in remaining if argument != action_name)
 
 
-def argument_is_flag(argument):
-    '''
-    Return True if the given argument looks like a flag, e.g. '--some-flag', as opposed to a
-    non-flag value.
-    '''
-    return isinstance(argument, str) and argument.startswith('--')
-
-
-def group_arguments_with_values(arguments):
-    '''
-    Given a sequence of arguments, return a sequence of tuples where each one contains either a
-    single argument (such as for a stand-alone flag) or a flag argument and its corresponding value.
-
-    For instance, given the following arguments sequence as input:
-
-      ('--foo', '--bar', '33', '--baz')
-
-    ... return the following output:
-
-      (('--foo',), ('--bar', '33'), ('--baz',))
-    '''
-    grouped_arguments = []
-    index = 0
-
-    while index < len(arguments):
-        this_argument = arguments[index]
-
-        try:
-            next_argument = arguments[index + 1]
-        except IndexError:
-            grouped_arguments.append((this_argument,))
-            break
-
-        if (
-            argument_is_flag(this_argument)
-            and not argument_is_flag(next_argument)
-            and next_argument not in ACTION_ALIASES
-        ):
-            grouped_arguments.append((this_argument, next_argument))
-            index += 2
-            continue
-
-        grouped_arguments.append((this_argument,))
-        index += 1
-
-    return tuple(grouped_arguments)
-
-
 def get_unparsable_arguments(remaining_action_arguments):
     '''
     Given a sequence of argument tuples (one per action parser that parsed arguments), determine the
@@ -172,21 +121,12 @@ def get_unparsable_arguments(remaining_action_arguments):
     if not remaining_action_arguments:
         return ()
 
-    grouped_action_arguments = tuple(
-        group_arguments_with_values(action_arguments)
-        for action_arguments in remaining_action_arguments
-    )
-
     return tuple(
-        itertools.chain.from_iterable(
-            argument_group
-            for argument_group in dict.fromkeys(
-                itertools.chain.from_iterable(grouped_action_arguments)
-            ).keys()
-            if all(
-                argument_group in action_arguments for action_arguments in grouped_action_arguments
-            )
-        )
+        argument
+        for argument in dict.fromkeys(
+            itertools.chain.from_iterable(remaining_action_arguments)
+        ).keys()
+        if all(argument in action_arguments for action_arguments in remaining_action_arguments)
     )
 
 
@@ -301,7 +241,7 @@ def make_parsers():
         '--config',
         dest='config_paths',
         action='append',
-        help=f"Configuration filename or directory, can specify flag multiple times, defaults to: -c {' -c '.join(unexpanded_config_paths)}",
+        help=f"Configuration filename or directory, can specify flag multiple times, defaults to: {' '.join(unexpanded_config_paths)}",
     )
     global_group.add_argument(
         '-n',
@@ -350,12 +290,12 @@ def make_parsers():
     global_group.add_argument(
         '--log-file-format',
         type=str,
-        help='Python format string used for log messages written to the log file',
+        help='Log format string used for log messages written to the log file',
     )
     global_group.add_argument(
         '--log-json',
         action='store_true',
-        help='Write Borg log messages and console output as one JSON object per log line instead of formatted text',
+        help='Write log messages and console output as one JSON object per log line instead of formatted text',
     )
     global_group.add_argument(
         '--override',
@@ -403,51 +343,51 @@ def make_parsers():
         metavar='',
         help='Specify zero or more actions. Defaults to create, prune, compact, and check. Use --help with action for details:',
     )
-    repo_create_parser = action_parsers.add_parser(
-        'repo-create',
-        aliases=ACTION_ALIASES['repo-create'],
+    rcreate_parser = action_parsers.add_parser(
+        'rcreate',
+        aliases=ACTION_ALIASES['rcreate'],
         help='Create a new, empty Borg repository',
         description='Create a new, empty Borg repository',
         add_help=False,
     )
-    repo_create_group = repo_create_parser.add_argument_group('repo-create arguments')
-    repo_create_group.add_argument(
+    rcreate_group = rcreate_parser.add_argument_group('rcreate arguments')
+    rcreate_group.add_argument(
         '-e',
         '--encryption',
         dest='encryption_mode',
         help='Borg repository encryption mode',
         required=True,
     )
-    repo_create_group.add_argument(
+    rcreate_group.add_argument(
         '--source-repository',
         '--other-repo',
         metavar='KEY_REPOSITORY',
         help='Path to an existing Borg repository whose key material should be reused [Borg 2.x+ only]',
     )
-    repo_create_group.add_argument(
+    rcreate_group.add_argument(
         '--repository',
-        help='Path of the new repository to create (must be already specified in a borgmatic configuration file), defaults to the configured repository if there is only one, quoted globs supported',
+        help='Path of the new repository to create (must be already specified in a borgmatic configuration file), defaults to the configured repository if there is only one',
     )
-    repo_create_group.add_argument(
+    rcreate_group.add_argument(
         '--copy-crypt-key',
         action='store_true',
         help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key [Borg 2.x+ only]',
     )
-    repo_create_group.add_argument(
+    rcreate_group.add_argument(
         '--append-only',
         action='store_true',
         help='Create an append-only repository',
     )
-    repo_create_group.add_argument(
+    rcreate_group.add_argument(
         '--storage-quota',
         help='Create a repository with a fixed storage quota',
     )
-    repo_create_group.add_argument(
+    rcreate_group.add_argument(
         '--make-parent-dirs',
         action='store_true',
         help='Create any missing parent directories of the repository directory',
     )
-    repo_create_group.add_argument(
+    rcreate_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
@@ -461,7 +401,7 @@ def make_parsers():
     transfer_group = transfer_parser.add_argument_group('transfer arguments')
     transfer_group.add_argument(
         '--repository',
-        help='Path of existing destination repository to transfer archives to, defaults to the configured repository if there is only one, quoted globs supported',
+        help='Path of existing destination repository to transfer archives to, defaults to the configured repository if there is only one',
     )
     transfer_group.add_argument(
         '--source-repository',
@@ -470,7 +410,7 @@ def make_parsers():
     )
     transfer_group.add_argument(
         '--archive',
-        help='Name or hash of a single archive to transfer (or "latest"), defaults to transferring all archives',
+        help='Name of single archive to transfer (or "latest"), defaults to transferring all archives',
     )
     transfer_group.add_argument(
         '--upgrader',
@@ -487,7 +427,7 @@ def make_parsers():
         '--match-archives',
         '--glob-archives',
         metavar='PATTERN',
-        help='Only transfer archives with names, hashes, or series matching this pattern',
+        help='Only transfer archives with names matching this pattern',
     )
     transfer_group.add_argument(
         '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
@@ -534,21 +474,14 @@ def make_parsers():
     prune_group = prune_parser.add_argument_group('prune arguments')
     prune_group.add_argument(
         '--repository',
-        help='Path of specific existing repository to prune (must be already specified in a borgmatic configuration file), quoted globs supported',
-    )
-    prune_group.add_argument(
-        '-a',
-        '--match-archives',
-        '--glob-archives',
-        metavar='PATTERN',
-        help='When pruning, only consider archives with names, hashes, or series matching this pattern',
+        help='Path of specific existing repository to prune (must be already specified in a borgmatic configuration file)',
     )
     prune_group.add_argument(
         '--stats',
         dest='stats',
         default=False,
         action='store_true',
-        help='Display statistics of the pruned archive [Borg 1 only]',
+        help='Display statistics of archive',
     )
     prune_group.add_argument(
         '--list', dest='list_archives', action='store_true', help='List archives kept/pruned'
@@ -585,7 +518,7 @@ def make_parsers():
     compact_group = compact_parser.add_argument_group('compact arguments')
     compact_group.add_argument(
         '--repository',
-        help='Path of specific existing repository to compact (must be already specified in a borgmatic configuration file), quoted globs supported',
+        help='Path of specific existing repository to compact (must be already specified in a borgmatic configuration file)',
     )
     compact_group.add_argument(
         '--progress',
@@ -621,7 +554,7 @@ def make_parsers():
     create_group = create_parser.add_argument_group('create arguments')
     create_group.add_argument(
         '--repository',
-        help='Path of specific existing repository to backup to (must be already specified in a borgmatic configuration file), quoted globs supported',
+        help='Path of specific existing repository to backup to (must be already specified in a borgmatic configuration file)',
     )
     create_group.add_argument(
         '--progress',
@@ -655,7 +588,7 @@ def make_parsers():
     check_group = check_parser.add_argument_group('check arguments')
     check_group.add_argument(
         '--repository',
-        help='Path of specific existing repository to check (must be already specified in a borgmatic configuration file), quoted globs supported',
+        help='Path of specific existing repository to check (must be already specified in a borgmatic configuration file)',
     )
     check_group.add_argument(
         '--progress',
@@ -671,17 +604,12 @@ def make_parsers():
         action='store_true',
         help='Attempt to repair any inconsistencies found (for interactive use)',
     )
-    check_group.add_argument(
-        '--max-duration',
-        metavar='SECONDS',
-        help='How long to check the repository before interrupting the check, defaults to no interruption',
-    )
     check_group.add_argument(
         '-a',
         '--match-archives',
         '--glob-archives',
         metavar='PATTERN',
-        help='Only check archives with names, hashes, or series matching this pattern',
+        help='Only check archives with names matching this pattern',
     )
     check_group.add_argument(
         '--only',
@@ -699,97 +627,6 @@ def make_parsers():
     )
     check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    delete_parser = action_parsers.add_parser(
-        'delete',
-        aliases=ACTION_ALIASES['delete'],
-        help='Delete an archive from a repository or delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
-        description='Delete an archive from a repository or delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
-        add_help=False,
-    )
-    delete_group = delete_parser.add_argument_group('delete arguments')
-    delete_group.add_argument(
-        '--repository',
-        help='Path of repository to delete or delete archives from, defaults to the configured repository if there is only one, quoted globs supported',
-    )
-    delete_group.add_argument(
-        '--archive',
-        help='Archive name, hash, or series to delete',
-    )
-    delete_group.add_argument(
-        '--list',
-        dest='list_archives',
-        action='store_true',
-        help='Show details for the deleted archives',
-    )
-    delete_group.add_argument(
-        '--stats',
-        action='store_true',
-        help='Display statistics for the deleted archives',
-    )
-    delete_group.add_argument(
-        '--cache-only',
-        action='store_true',
-        help='Delete only the local cache for the given repository',
-    )
-    delete_group.add_argument(
-        '--force',
-        action='count',
-        help='Force deletion of corrupted archives, can be given twice if once does not work',
-    )
-    delete_group.add_argument(
-        '--keep-security-info',
-        action='store_true',
-        help='Do not delete the local security info when deleting a repository',
-    )
-    delete_group.add_argument(
-        '--save-space',
-        action='store_true',
-        help='Work slower, but using less space [Not supported in Borg 2.x+]',
-    )
-    delete_group.add_argument(
-        '--checkpoint-interval',
-        type=int,
-        metavar='SECONDS',
-        help='Write a checkpoint at the given interval, defaults to 1800 seconds (30 minutes)',
-    )
-    delete_group.add_argument(
-        '-a',
-        '--match-archives',
-        '--glob-archives',
-        metavar='PATTERN',
-        help='Only delete archives with names, hashes, or series matching this pattern',
-    )
-    delete_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
-    )
-    delete_group.add_argument(
-        '--first', metavar='N', help='Delete first N archives after other filters are applied'
-    )
-    delete_group.add_argument(
-        '--last', metavar='N', help='Delete last N archives after other filters are applied'
-    )
-    delete_group.add_argument(
-        '--oldest',
-        metavar='TIMESPAN',
-        help='Delete archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]',
-    )
-    delete_group.add_argument(
-        '--newest',
-        metavar='TIMESPAN',
-        help='Delete archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]',
-    )
-    delete_group.add_argument(
-        '--older',
-        metavar='TIMESPAN',
-        help='Delete archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
-    )
-    delete_group.add_argument(
-        '--newer',
-        metavar='TIMESPAN',
-        help='Delete archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
-    )
-    delete_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
-
     extract_parser = action_parsers.add_parser(
         'extract',
         aliases=ACTION_ALIASES['extract'],
@@ -800,10 +637,10 @@ def make_parsers():
     extract_group = extract_parser.add_argument_group('extract arguments')
     extract_group.add_argument(
         '--repository',
-        help='Path of repository to extract, defaults to the configured repository if there is only one, quoted globs supported',
+        help='Path of repository to extract, defaults to the configured repository if there is only one',
     )
     extract_group.add_argument(
-        '--archive', help='Name or hash of a single archive to extract (or "latest")', required=True
+        '--archive', help='Name of archive to extract (or "latest")', required=True
     )
     extract_group.add_argument(
         '--path',
@@ -862,30 +699,16 @@ def make_parsers():
     )
     config_bootstrap_group.add_argument(
         '--repository',
-        help='Path of repository to extract config files from, quoted globs supported',
+        help='Path of repository to extract config files from',
         required=True,
     )
-    config_bootstrap_group.add_argument(
-        '--local-path',
-        help='Alternate Borg local executable. Defaults to "borg"',
-        default='borg',
-    )
-    config_bootstrap_group.add_argument(
-        '--remote-path',
-        help='Alternate Borg remote executable. Defaults to "borg"',
-        default='borg',
-    )
-    config_bootstrap_group.add_argument(
-        '--user-runtime-directory',
-        help='Path used for temporary runtime data like bootstrap metadata. Defaults to $XDG_RUNTIME_DIR or $TMPDIR or $TEMP or /var/run/$UID',
-    )
     config_bootstrap_group.add_argument(
         '--borgmatic-source-directory',
-        help='Deprecated. Path formerly used for temporary runtime data like bootstrap metadata. Defaults to ~/.borgmatic',
+        help='Path that stores the config files used to create an archive and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic',
     )
     config_bootstrap_group.add_argument(
         '--archive',
-        help='Name or hash of a single archive to extract config files from, defaults to "latest"',
+        help='Name of archive to extract config files from, defaults to "latest"',
         default='latest',
     )
     config_bootstrap_group.add_argument(
@@ -974,10 +797,10 @@ def make_parsers():
     export_tar_group = export_tar_parser.add_argument_group('export-tar arguments')
     export_tar_group.add_argument(
         '--repository',
-        help='Path of repository to export from, defaults to the configured repository if there is only one, quoted globs supported',
+        help='Path of repository to export from, defaults to the configured repository if there is only one',
     )
     export_tar_group.add_argument(
-        '--archive', help='Name or hash of a single archive to export (or "latest")', required=True
+        '--archive', help='Name of archive to export (or "latest")', required=True
     )
     export_tar_group.add_argument(
         '--path',
@@ -1020,11 +843,9 @@ def make_parsers():
     mount_group = mount_parser.add_argument_group('mount arguments')
     mount_group.add_argument(
         '--repository',
-        help='Path of repository to use, defaults to the configured repository if there is only one, quoted globs supported',
-    )
-    mount_group.add_argument(
-        '--archive', help='Name or hash of a single archive to mount (or "latest")'
+        help='Path of repository to use, defaults to the configured repository if there is only one',
     )
+    mount_group.add_argument('--archive', help='Name of archive to mount (or "latest")')
     mount_group.add_argument(
         '--mount-point',
         metavar='PATH',
@@ -1094,43 +915,6 @@ def make_parsers():
     )
     umount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    repo_delete_parser = action_parsers.add_parser(
-        'repo-delete',
-        aliases=ACTION_ALIASES['repo-delete'],
-        help='Delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
-        description='Delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
-        add_help=False,
-    )
-    repo_delete_group = repo_delete_parser.add_argument_group('delete arguments')
-    repo_delete_group.add_argument(
-        '--repository',
-        help='Path of repository to delete, defaults to the configured repository if there is only one, quoted globs supported',
-    )
-    repo_delete_group.add_argument(
-        '--list',
-        dest='list_archives',
-        action='store_true',
-        help='Show details for the archives in the given repository',
-    )
-    repo_delete_group.add_argument(
-        '--force',
-        action='count',
-        help='Force deletion of corrupted archives, can be given twice if once does not work',
-    )
-    repo_delete_group.add_argument(
-        '--cache-only',
-        action='store_true',
-        help='Delete only the local cache for the given repository',
-    )
-    repo_delete_group.add_argument(
-        '--keep-security-info',
-        action='store_true',
-        help='Do not delete the local security info when deleting a repository',
-    )
-    repo_delete_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
-    )
-
     restore_parser = action_parsers.add_parser(
         'restore',
         aliases=ACTION_ALIASES['restore'],
@@ -1141,12 +925,10 @@ def make_parsers():
     restore_group = restore_parser.add_argument_group('restore arguments')
     restore_group.add_argument(
         '--repository',
-        help='Path of repository to restore from, defaults to the configured repository if there is only one, quoted globs supported',
+        help='Path of repository to restore from, defaults to the configured repository if there is only one',
     )
     restore_group.add_argument(
-        '--archive',
-        help='Name or hash of a single archive to restore from (or "latest")',
-        required=True,
+        '--archive', help='Name of archive to restore from (or "latest")', required=True
     )
     restore_group.add_argument(
         '--data-source',
@@ -1154,7 +936,7 @@ def make_parsers():
         metavar='NAME',
         dest='data_sources',
         action='append',
-        help="Name of data source (e.g. database) to restore from the archive, must be defined in borgmatic's configuration, can specify the flag multiple times, defaults to all data sources in the archive",
+        help="Name of data source (e.g. database) to restore from archive, must be defined in borgmatic's configuration, can specify flag multiple times, defaults to all data sources in the archive",
     )
     restore_group.add_argument(
         '--schema',
@@ -1183,90 +965,69 @@ def make_parsers():
         '--restore-path',
         help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
     )
-    restore_group.add_argument(
-        '--original-hostname',
-        help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps',
-    )
-    restore_group.add_argument(
-        '--original-port',
-        type=int,
-        help="The port where the dump to restore came from (if that port is in borgmatic's configuration), only necessary if you need to disambiguate dumps",
-    )
-    restore_group.add_argument(
-        '--hook',
-        help='The name of the data source hook for the dump to restore, only necessary if you need to disambiguate dumps',
-    )
     restore_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    repo_list_parser = action_parsers.add_parser(
-        'repo-list',
-        aliases=ACTION_ALIASES['repo-list'],
+    rlist_parser = action_parsers.add_parser(
+        'rlist',
+        aliases=ACTION_ALIASES['rlist'],
         help='List repository',
         description='List the archives in a repository',
         add_help=False,
     )
-    repo_list_group = repo_list_parser.add_argument_group('repo-list arguments')
-    repo_list_group.add_argument(
+    rlist_group = rlist_parser.add_argument_group('rlist arguments')
+    rlist_group.add_argument(
         '--repository',
-        help='Path of repository to list, defaults to the configured repositories, quoted globs supported',
+        help='Path of repository to list, defaults to the configured repositories',
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '--short', default=False, action='store_true', help='Output only archive names'
     )
-    repo_list_group.add_argument('--format', help='Format for archive listing')
-    repo_list_group.add_argument(
+    rlist_group.add_argument('--format', help='Format for archive listing')
+    rlist_group.add_argument(
         '--json', default=False, action='store_true', help='Output results as JSON'
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '-a',
         '--match-archives',
         '--glob-archives',
         metavar='PATTERN',
-        help='Only list archive names, hashes, or series matching this pattern',
+        help='Only list archive names matching this pattern',
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '--first', metavar='N', help='List first N archives after other filters are applied'
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '--last', metavar='N', help='List last N archives after other filters are applied'
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '--oldest',
         metavar='TIMESPAN',
         help='List archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]',
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '--newest',
         metavar='TIMESPAN',
         help='List archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]',
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '--older',
         metavar='TIMESPAN',
         help='List archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
     )
-    repo_list_group.add_argument(
+    rlist_group.add_argument(
         '--newer',
         metavar='TIMESPAN',
         help='List archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
     )
-    repo_list_group.add_argument(
-        '--deleted',
-        default=False,
-        action='store_true',
-        help="List only deleted archives that haven't yet been compacted [Borg 2.x+ only]",
-    )
-    repo_list_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
-    )
+    rlist_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
     list_parser = action_parsers.add_parser(
         'list',
@@ -1278,11 +1039,9 @@ def make_parsers():
     list_group = list_parser.add_argument_group('list arguments')
     list_group.add_argument(
         '--repository',
-        help='Path of repository containing archive to list, defaults to the configured repositories, quoted globs supported',
-    )
-    list_group.add_argument(
-        '--archive', help='Name or hash of a single archive to list (or "latest")'
+        help='Path of repository containing archive to list, defaults to the configured repositories',
     )
+    list_group.add_argument('--archive', help='Name of the archive to list (or "latest")')
     list_group.add_argument(
         '--path',
         metavar='PATH',
@@ -1337,24 +1096,22 @@ def make_parsers():
     )
     list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    repo_info_parser = action_parsers.add_parser(
-        'repo-info',
-        aliases=ACTION_ALIASES['repo-info'],
+    rinfo_parser = action_parsers.add_parser(
+        'rinfo',
+        aliases=ACTION_ALIASES['rinfo'],
         help='Show repository summary information such as disk space used',
         description='Show repository summary information such as disk space used',
         add_help=False,
     )
-    repo_info_group = repo_info_parser.add_argument_group('repo-info arguments')
-    repo_info_group.add_argument(
+    rinfo_group = rinfo_parser.add_argument_group('rinfo arguments')
+    rinfo_group.add_argument(
         '--repository',
-        help='Path of repository to show info for, defaults to the configured repository if there is only one, quoted globs supported',
+        help='Path of repository to show info for, defaults to the configured repository if there is only one',
     )
-    repo_info_group.add_argument(
+    rinfo_group.add_argument(
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )
-    repo_info_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
-    )
+    rinfo_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
     info_parser = action_parsers.add_parser(
         'info',
@@ -1366,11 +1123,9 @@ def make_parsers():
     info_group = info_parser.add_argument_group('info arguments')
     info_group.add_argument(
         '--repository',
-        help='Path of repository containing archive to show info for, defaults to the configured repository if there is only one, quoted globs supported',
-    )
-    info_group.add_argument(
-        '--archive', help='Archive name, hash, or series to show info for (or "latest")'
+        help='Path of repository containing archive to show info for, defaults to the configured repository if there is only one',
     )
+    info_group.add_argument('--archive', help='Name of archive to show info for (or "latest")')
     info_group.add_argument(
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )
@@ -1384,7 +1139,7 @@ def make_parsers():
         '--match-archives',
         '--glob-archives',
         metavar='PATTERN',
-        help='Only show info for archive names, hashes, or series matching this pattern',
+        help='Only show info for archive names matching this pattern',
     )
     info_group.add_argument(
         '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
@@ -1429,7 +1184,7 @@ def make_parsers():
     break_lock_group = break_lock_parser.add_argument_group('break-lock arguments')
     break_lock_group.add_argument(
         '--repository',
-        help='Path of repository to break the lock for, defaults to the configured repository if there is only one, quoted globs supported',
+        help='Path of repository to break the lock for, defaults to the configured repository if there is only one',
     )
     break_lock_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
@@ -1469,7 +1224,7 @@ def make_parsers():
     )
     key_export_group.add_argument(
         '--repository',
-        help='Path of repository to export the key for, defaults to the configured repository if there is only one, quoted globs supported',
+        help='Path of repository to export the key for, defaults to the configured repository if there is only one',
     )
     key_export_group.add_argument(
         '--path',
@@ -1480,94 +1235,6 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    key_import_parser = key_parsers.add_parser(
-        'import',
-        help='Import a copy of the repository key from backup',
-        description='Import a copy of the repository key from backup',
-        add_help=False,
-    )
-    key_import_group = key_import_parser.add_argument_group('key import arguments')
-    key_import_group.add_argument(
-        '--paper',
-        action='store_true',
-        help='Import interactively from a backup done with --paper',
-    )
-    key_import_group.add_argument(
-        '--repository',
-        help='Path of repository to import the key from, defaults to the configured repository if there is only one, quoted globs supported',
-    )
-    key_import_group.add_argument(
-        '--path',
-        metavar='PATH',
-        help='Path to import the key from backup, defaults to stdin',
-    )
-    key_import_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
-    )
-
-    key_change_passphrase_parser = key_parsers.add_parser(
-        'change-passphrase',
-        help='Change the passphrase protecting the repository key',
-        description='Change the passphrase protecting the repository key',
-        add_help=False,
-    )
-    key_change_passphrase_group = key_change_passphrase_parser.add_argument_group(
-        'key change-passphrase arguments'
-    )
-    key_change_passphrase_group.add_argument(
-        '--repository',
-        help='Path of repository to change the passphrase for, defaults to the configured repository if there is only one, quoted globs supported',
-    )
-    key_change_passphrase_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
-    )
-
-    recreate_parser = action_parsers.add_parser(
-        'recreate',
-        aliases=ACTION_ALIASES['recreate'],
-        help='Recreate an archive in a repository',
-        description='Recreate an archive in a repository',
-        add_help=False,
-    )
-    recreate_group = recreate_parser.add_argument_group('recreate arguments')
-    recreate_group.add_argument(
-        '--repository',
-        help='Path of repository containing archive to recreate, defaults to the configured repository if there is only one, quoted globs supported',
-    )
-    recreate_group.add_argument(
-        '--archive',
-        help='Archive name, hash, or series to recreate',
-    )
-    recreate_group.add_argument(
-        '--list', dest='list', action='store_true', help='Show per-file details'
-    )
-    recreate_group.add_argument(
-        '--target',
-        metavar='TARGET',
-        help='Create a new archive from the specified archive (via --archive), without replacing it',
-    )
-    recreate_group.add_argument(
-        '--comment',
-        metavar='COMMENT',
-        help='Add a comment text to the archive or, if an archive is not provided, to all matching archives',
-    )
-    recreate_group.add_argument(
-        '--timestamp',
-        metavar='TIMESTAMP',
-        help='Manually override the archive creation date/time (UTC)',
-    )
-    recreate_group.add_argument(
-        '-a',
-        '--match-archives',
-        '--glob-archives',
-        dest='match_archives',
-        metavar='PATTERN',
-        help='Only consider archive names, hashes, or series matching this pattern',
-    )
-    recreate_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
-    )
-
     borg_parser = action_parsers.add_parser(
         'borg',
         aliases=ACTION_ALIASES['borg'],
@@ -1578,11 +1245,9 @@ def make_parsers():
     borg_group = borg_parser.add_argument_group('borg arguments')
     borg_group.add_argument(
         '--repository',
-        help='Path of repository to pass to Borg, defaults to the configured repositories, quoted globs supported',
-    )
-    borg_group.add_argument(
-        '--archive', help='Archive name, hash, or series to pass to Borg (or "latest")'
+        help='Path of repository to pass to Borg, defaults to the configured repositories',
     )
+    borg_group.add_argument('--archive', help='Name of archive to pass to Borg (or "latest")')
     borg_group.add_argument(
         '--',
         metavar='OPTION',
@@ -1641,9 +1306,9 @@ def parse_arguments(*unparsed_arguments):
         )
 
     if (
-        ('list' in arguments and 'repo-info' in arguments and arguments['list'].json)
+        ('list' in arguments and 'rinfo' in arguments and arguments['list'].json)
         or ('list' in arguments and 'info' in arguments and arguments['list'].json)
-        or ('repo-info' in arguments and 'info' in arguments and arguments['repo-info'].json)
+        or ('rinfo' in arguments and 'info' in arguments and arguments['rinfo'].json)
     ):
         raise ValueError('With the --json flag, multiple actions cannot be used together.')
 
@@ -1661,11 +1326,9 @@ def parse_arguments(*unparsed_arguments):
             'With the list action, only one of --prefix or --match-archives flags can be used.'
         )
 
-    if 'repo-list' in arguments and (
-        arguments['repo-list'].prefix and arguments['repo-list'].match_archives
-    ):
+    if 'rlist' in arguments and (arguments['rlist'].prefix and arguments['rlist'].match_archives):
         raise ValueError(
-            'With the repo-list action, only one of --prefix or --match-archives flags can be used.'
+            'With the rlist action, only one of --prefix or --match-archives flags can be used.'
         )
 
     if 'info' in arguments and (

borgmatic/config/constants.py

@@ -50,15 +50,12 @@ def apply_constants(value, constants, shell_escape=False):
             value[index] = apply_constants(list_value, constants, shell_escape)
     elif isinstance(value, dict):
         for option_name, option_value in value.items():
-            value[option_name] = apply_constants(
-                option_value,
-                constants,
-                shell_escape=(
-                    shell_escape
-                    or option_name.startswith('before_')
-                    or option_name.startswith('after_')
-                    or option_name == 'on_error'
-                ),
+            shell_escape = (
+                shell_escape
+                or option_name.startswith('before_')
+                or option_name.startswith('after_')
+                or option_name == 'on_error'
             )
+            value[option_name] = apply_constants(option_value, constants, shell_escape)
 
     return value

borgmatic/config/generate.py

@@ -1,6 +1,5 @@
 import collections
 import io
-import itertools
 import os
 import re
 
@@ -25,65 +24,41 @@ def insert_newline_before_comment(config, field_name):
 def get_properties(schema):
     '''
     Given a schema dict, return its properties. But if it's got sub-schemas with multiple different
-    potential properties, returned their merged properties instead (interleaved so the first
-    properties of each sub-schema come first). The idea is that the user should see all possible
-    options even if they're not all possible together.
+    potential properties, returned their merged properties instead.
     '''
     if 'oneOf' in schema:
         return dict(
-            item
-            for item in itertools.chain(
-                *itertools.zip_longest(
-                    *[sub_schema['properties'].items() for sub_schema in schema['oneOf']]
-                )
-            )
-            if item is not None
+            collections.ChainMap(*[sub_schema['properties'] for sub_schema in schema['oneOf']])
         )
 
     return schema['properties']
 
 
-def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False):
+def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     '''
-    Given a loaded configuration schema and a source configuration, generate and return sample
-    config for the schema. Include comments for each option based on the schema "description".
-
-    If a source config is given, walk it alongside the given schema so that both can be taken into
-    account when commenting out particular options in add_comments_to_configuration_object().
+    Given a loaded configuration schema, generate and return sample config for it. Include comments
+    for each option based on the schema "description".
     '''
     schema_type = schema.get('type')
     example = schema.get('example')
-
     if example is not None:
         return example
 
-    if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type):
+    if schema_type == 'array':
         config = ruamel.yaml.comments.CommentedSeq(
-            [
-                schema_to_sample_configuration(
-                    schema['items'], source_config, level, parent_is_sequence=True
-                )
-            ]
+            [schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
         )
         add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
-    elif schema_type == 'object' or (isinstance(schema_type, list) and 'object' in schema_type):
-        if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict):
-            source_config = dict(collections.ChainMap(*source_config))
-
+    elif schema_type == 'object':
         config = ruamel.yaml.comments.CommentedMap(
             [
-                (
-                    field_name,
-                    schema_to_sample_configuration(
-                        sub_schema, (source_config or {}).get(field_name, {}), level + 1
-                    ),
-                )
+                (field_name, schema_to_sample_configuration(sub_schema, level + 1))
                 for field_name, sub_schema in get_properties(schema).items()
             ]
         )
         indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)