Compare commits
198 Commits
Author | SHA1 | Date |
---|---|---|
Dan Helfman | fecae39fcd | |
Dan Helfman | 38bc4fbfe2 | |
Dan Helfman | 92ed7573d4 | |
Dan Helfman | 80f0e92462 | |
Dan Helfman | 5f10b1b2ca | |
Dan Helfman | 4f83b1e6b3 | |
Codimp | 15d5a687fb | |
Codimp | eb1fce3787 | |
Dan Helfman | 7f735cbe59 | |
Dan Helfman | a690ea4016 | |
Dan Helfman | 7a110c7acd | |
estebanthilliez | 407bb33359 | |
estebanthilliez | 4b7f7bba04 | |
estebanthilliez | cfdc0a1f2a | |
Dan Helfman | f926055e67 | |
Dan Helfman | 058af95d70 | |
Dan Helfman | 54facdc391 | |
estebanthi | 2e4c0cc7e7 | |
Dan Helfman | cb2fd7c5e8 | |
Dan Helfman | 94133cc8b1 | |
Dan Helfman | dcec89be90 | |
Dan Helfman | fefd5d1d0e | |
Dan Helfman | 163c37d77f | |
Dan Helfman | b0e49ebce0 | |
Dan Helfman | 7e51c41ebf | |
Dan Helfman | f9182514d8 | |
Dan Helfman | 7700b87b60 | |
Dan Helfman | 75bdbe6087 | |
Dan Helfman | d243a8c836 | |
Dan Helfman | 4c2eb2bfe3 | |
Dan Helfman | 89ce060dbd | |
Dan Helfman | ad7dcb4615 | |
Dan Helfman | 6680aece5a | |
Dan Helfman | 57eb93760f | |
Dan Helfman | f21a2c06e3 | |
Dan Helfman | 2212539cb0 | |
Dan Helfman | 36d10fecb1 | |
Dan Helfman | 3ecd0e731e | |
Dan Helfman | ecf5a7e294 | |
Dan Helfman | 893fbcf9ff | |
Dan Helfman | f8f6560502 | |
Dan Helfman | 8c301ba688 | |
Dan Helfman | 035e96156a | |
Dan Helfman | a08c7fc77a | |
Dan Helfman | cf9e387811 | |
Dan Helfman | e37224606a | |
Dan Helfman | 9647301b99 | |
Dan Helfman | a0e5dbff96 | |
Dan Helfman | 86117edccf | |
Dan Helfman | 440f3eeb63 | |
Dan Helfman | 181051eae1 | |
Dan Helfman | ec0ee971ed | |
Dan Helfman | b83ffa0cf6 | |
Dan Helfman | cf88665d37 | |
Dan Helfman | b233adba63 | |
Dan Helfman | 018f5e3315 | |
Dan Helfman | 284f26b49d | |
Dan Helfman | 11b437794e | |
Dan Helfman | 0665b50d57 | |
Dan Helfman | 0586b80e5b | |
Dan Helfman | 272a7b4866 | |
Dan Helfman | 98d4a59459 | |
Dan Helfman | 744139cf97 | |
Dan Helfman | 1339509e9b | |
Dan Helfman | e14f61415b | |
Dan Helfman | 98cf8f7e20 | |
Dan Helfman | 5f16b64639 | |
Dan Helfman | fe62a81151 | |
Dan Helfman | 585b1573ae | |
Dan Helfman | 141ba2771d | |
Dan Helfman | a527f76d08 | |
Dan Helfman | a97c68b4c8 | |
Dan Helfman | ef07005a75 | |
Dan Helfman | 43c7c3b6be | |
Dan Helfman | 2f6ad9d173 | |
Dan Helfman | 16bc0de3fb | |
shivansh02 | 458d157e62 | |
shivansh02 | 40c3a28620 | |
Dan Helfman | 60107f1ee8 | |
Dan Helfman | a1153a21fa | |
shivansh02 | b6cb7da98e | |
shivansh02 | 9e3d19a406 | |
shivansh02 | 2b755d8ade | |
shivansh02 | 925f99cfef | |
Dan Helfman | c9f20eb260 | |
Dan Helfman | f4744826fe | |
Dan Helfman | 5586aab967 | |
Dan Helfman | 6fa5dff79b | |
Dan Helfman | 75d11aa9cd | |
Dan Helfman | ad1d104d65 | |
Dan Helfman | 009062128d | |
Dan Helfman | e9813d2539 | |
Dan Helfman | f9998b50e8 | |
Dan Helfman | 5f921a7f80 | |
Dan Helfman | abf2b3a8c7 | |
Dan Helfman | 34f3c2bb16 | |
Dan Helfman | 4d79f582df | |
Dan Helfman | 63198088c4 | |
Dan Helfman | 3c22a8ec16 | |
Dan Helfman | ca49109ce7 | |
Dan Helfman | 6a7f71f92f | |
Dan Helfman | 5f3dc1cfb0 | |
Dan Helfman | f2023aed22 | |
Dan Helfman | a03c2744e5 | |
axel simon | 4176532317 | |
Dan Helfman | 9d6025e902 | |
Dan Helfman | cf739bc997 | |
Dan Helfman | 84823dfb91 | |
Dan Helfman | 20cf0f7089 | |
Dan Helfman | 67af0f5734 | |
Dan Helfman | e80e0a253c | |
Dan Helfman | 72587a3b72 | |
Dan Helfman | 8b49a59aff | |
Dan Helfman | e120dff9ff | |
Tobias Hodapp | 257678b66f | |
Tobias Hodapp | 422c5e32f4 | |
Dan Helfman | c34ad7dde7 | |
Dan Helfman | fdb353d358 | |
Dan Helfman | 3b99f7c75a | |
Dan Helfman | 8b9abc6cf8 | |
Dan Helfman | da034c316a | |
Dan Helfman | 08d01d8bcd | |
Dan Helfman | eef69e23ee | |
Dan Helfman | 26bb54a9dd | |
Dan Helfman | 715e2ac127 | |
Dan Helfman | f39cea4abf | |
Dan Helfman | 22101bdd49 | |
Dan Helfman | 13cf863d89 | |
Dan Helfman | dcf25fa041 | |
Dan Helfman | 12b75f9075 | |
Dan Helfman | 9baf06a2f7 | |
tdltdc | 56302e22cd | |
Dan Helfman | 6cc93c4eb9 | |
Dan Helfman | 2da43239f6 | |
debuglevel | 4beef36d3c | |
debuglevel | eacfbd742b | |
debuglevel | 82a85986b6 | |
Dan Helfman | ef448e2dd1 | |
Dan Helfman | c3efe1b90e | |
Dan Helfman | d85c1ee216 | |
Dan Helfman | b47088067c | |
Dan Helfman | c5732aa4fc | |
Dan Helfman | a0323d9d6c | |
Dan Helfman | 8ad7b473f1 | |
Dan Helfman | 895a0ccb3c | |
Dan Helfman | 257ab77bea | |
Dan Helfman | dccaa4014b | |
David Härdeman | 2f3c0bec5b | |
Dan Helfman | 487d8ffd32 | |
Dan Helfman | 30523a7c89 | |
Dan Helfman | 77b1907d03 | |
Dan Helfman | 09594c85bf | |
Dan Helfman | e07efdf68f | |
Dan Helfman | 1fed44f905 | |
Dan Helfman | c687dafdd2 | |
Dan Helfman | 3eff2c4248 | |
Dan Helfman | d94fdb6faf | |
Pim Kunis | a83282faf0 | |
Dan Helfman | e7169f6fb2 | |
Dan Helfman | 9587fc2366 | |
Dan Helfman | 5f06884d5a | |
Dan Helfman | f011431463 | |
Dan Helfman | 9e14f209f1 | |
Dan Helfman | 9d34d2eec5 | |
Pim Kunis | 7a9625cd44 | |
Pim Kunis | 4763c323d0 | |
Pim Kunis | eaa22be3db | |
Pim Kunis | a587e207f9 | |
Pim Kunis | db8079b699 | |
Pim Kunis | 5a989826a1 | |
Pim Kunis | 21f4266273 | |
Pim Kunis | e7252c7545 | |
Pim Kunis | 86011c8418 | |
Pim Kunis | f3295ccb4a | |
Dan Helfman | cacb81f086 | |
Dan Helfman | 06c2154e6a | |
Dan Helfman | ac1e1a9407 | |
Dan Helfman | 10933fd55b | |
Dan Helfman | af422ad705 | |
Dan Helfman | d9d35491fb | |
Dan Helfman | b540e63c0e | |
Dan Helfman | 5a56208922 | |
Dan Helfman | 5912769273 | |
Dan Helfman | bac2aabe66 | |
Dan Helfman | 9f3328781b | |
Dan Helfman | 0205748db8 | |
Dan Helfman | d0a8251ad2 | |
Dan Helfman | 32019ea8f3 | |
Dan Helfman | fa9a061033 | |
Dan Helfman | b3d2560563 | |
Dan Helfman | 4b4f56da42 | |
Dan Helfman | b96d1898f7 | |
Tobias Hodapp | 099a712e53 | |
Tobias Hodapp | 9e2674ea5a | |
Tobias Hodapp | 7e419ec995 | |
Tobias Hodapp | a3edf757ee | |
Tobias Hodapp | e576403b64 | |
Dan Helfman | 7313430178 |
97
.drone.yml
97
.drone.yml
|
@ -1,97 +0,0 @@
|
||||||
---
|
|
||||||
kind: pipeline
|
|
||||||
name: python-3-8-alpine-3-13
|
|
||||||
|
|
||||||
services:
|
|
||||||
- name: postgresql
|
|
||||||
image: docker.io/postgres:13.1-alpine
|
|
||||||
environment:
|
|
||||||
POSTGRES_PASSWORD: test
|
|
||||||
POSTGRES_DB: test
|
|
||||||
- name: postgresql2
|
|
||||||
image: docker.io/postgres:13.1-alpine
|
|
||||||
environment:
|
|
||||||
POSTGRES_PASSWORD: test2
|
|
||||||
POSTGRES_DB: test
|
|
||||||
commands:
|
|
||||||
- docker-entrypoint.sh -p 5433
|
|
||||||
- name: mariadb
|
|
||||||
image: docker.io/mariadb:10.11.4
|
|
||||||
environment:
|
|
||||||
MARIADB_ROOT_PASSWORD: test
|
|
||||||
MARIADB_DATABASE: test
|
|
||||||
- name: mariadb2
|
|
||||||
image: docker.io/mariadb:10.11.4
|
|
||||||
environment:
|
|
||||||
MARIADB_ROOT_PASSWORD: test2
|
|
||||||
MARIADB_DATABASE: test
|
|
||||||
commands:
|
|
||||||
- docker-entrypoint.sh --port=3307
|
|
||||||
- name: not-actually-mysql
|
|
||||||
image: docker.io/mariadb:10.11.4
|
|
||||||
environment:
|
|
||||||
MARIADB_ROOT_PASSWORD: test
|
|
||||||
MARIADB_DATABASE: test
|
|
||||||
- name: not-actually-mysql2
|
|
||||||
image: docker.io/mariadb:10.11.4
|
|
||||||
environment:
|
|
||||||
MARIADB_ROOT_PASSWORD: test2
|
|
||||||
MARIADB_DATABASE: test
|
|
||||||
commands:
|
|
||||||
- docker-entrypoint.sh --port=3307
|
|
||||||
- name: mongodb
|
|
||||||
image: docker.io/mongo:5.0.5
|
|
||||||
environment:
|
|
||||||
MONGO_INITDB_ROOT_USERNAME: root
|
|
||||||
MONGO_INITDB_ROOT_PASSWORD: test
|
|
||||||
- name: mongodb2
|
|
||||||
image: docker.io/mongo:5.0.5
|
|
||||||
environment:
|
|
||||||
MONGO_INITDB_ROOT_USERNAME: root2
|
|
||||||
MONGO_INITDB_ROOT_PASSWORD: test2
|
|
||||||
commands:
|
|
||||||
- docker-entrypoint.sh --port=27018
|
|
||||||
|
|
||||||
clone:
|
|
||||||
skip_verify: true
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: build
|
|
||||||
image: docker.io/alpine:3.13
|
|
||||||
environment:
|
|
||||||
TEST_CONTAINER: true
|
|
||||||
pull: always
|
|
||||||
commands:
|
|
||||||
- scripts/run-full-tests
|
|
||||||
---
|
|
||||||
kind: pipeline
|
|
||||||
name: documentation
|
|
||||||
type: exec
|
|
||||||
|
|
||||||
platform:
|
|
||||||
os: linux
|
|
||||||
arch: amd64
|
|
||||||
|
|
||||||
clone:
|
|
||||||
skip_verify: true
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: build
|
|
||||||
environment:
|
|
||||||
USERNAME:
|
|
||||||
from_secret: docker_username
|
|
||||||
PASSWORD:
|
|
||||||
from_secret: docker_password
|
|
||||||
IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
|
|
||||||
commands:
|
|
||||||
- podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org
|
|
||||||
- podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
|
|
||||||
- podman push "$IMAGE_NAME"
|
|
||||||
|
|
||||||
trigger:
|
|
||||||
repo:
|
|
||||||
- borgmatic-collective/borgmatic
|
|
||||||
branch:
|
|
||||||
- main
|
|
||||||
event:
|
|
||||||
- push
|
|
|
@ -1 +1 @@
|
||||||
blank_issues_enabled: false
|
blank_issues_enabled: true
|
||||||
|
|
|
@ -0,0 +1,28 @@
|
||||||
|
name: build
|
||||||
|
run-name: ${{ gitea.actor }} is building
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: host
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- run: scripts/run-end-to-end-tests
|
||||||
|
|
||||||
|
docs:
|
||||||
|
needs: [test]
|
||||||
|
runs-on: host
|
||||||
|
env:
|
||||||
|
IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- run: podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org
|
||||||
|
env:
|
||||||
|
USERNAME: "${{ secrets.REGISTRY_USERNAME }}"
|
||||||
|
PASSWORD: "${{ secrets.REGISTRY_PASSWORD }}"
|
||||||
|
- run: podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
|
||||||
|
- run: podman push "$IMAGE_NAME"
|
143
NEWS
143
NEWS
|
@ -1,3 +1,138 @@
|
||||||
|
1.8.12.dev0
|
||||||
|
* #860: Fix interaction between environment variable interpolation in constants and shell escaping.
|
||||||
|
|
||||||
|
1.8.11
|
||||||
|
* #815: Add optional Healthchecks auto-provisioning via "create_slug" option.
|
||||||
|
* #851: Fix lack of file extraction when using "extract --strip-components all" on a path with a
|
||||||
|
leading slash.
|
||||||
|
* #854: Fix a traceback when the "data" consistency check is used.
|
||||||
|
* #857: Fix a traceback with "check --only spot" when the "spot" check is unconfigured.
|
||||||
|
|
||||||
|
1.8.10
|
||||||
|
* #656 (beta): Add a "spot" consistency check that compares file counts and contents between your
|
||||||
|
source files and the latest archive, ensuring they fall within configured tolerances. This can
|
||||||
|
catch problems like incorrect excludes, inadvertent deletes, files changed by malware, etc. See
|
||||||
|
the documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#spot-check
|
||||||
|
* #779: When "--match-archives *" is used with "check" action, don't skip Borg's orphaned objects
|
||||||
|
check.
|
||||||
|
* #842: When a command hook exits with a soft failure, ping the log and finish states for any
|
||||||
|
configured monitoring hooks.
|
||||||
|
* #843: Add documentation link to Loki dashboard for borgmatic:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook
|
||||||
|
* #847: Fix "--json" error when Borg includes non-JSON warnings in JSON output.
|
||||||
|
* #848: SECURITY: Mask the password when logging a MongoDB dump or restore command.
|
||||||
|
* Fix handling of the NO_COLOR environment variable to ignore an empty value.
|
||||||
|
* Add documentation about backing up containerized databases by configuring borgmatic to exec into
|
||||||
|
a container to run a dump command:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
|
||||||
|
|
||||||
|
1.8.9
|
||||||
|
* #311: Add custom dump/restore command options for MySQL and MariaDB.
|
||||||
|
* #811: Add an "access_token" option to the ntfy monitoring hook for authenticating
|
||||||
|
without username/password.
|
||||||
|
* #827: When the "--json" flag is given, suppress console escape codes so as not to
|
||||||
|
interfere with JSON output.
|
||||||
|
* #829: Fix "--override" values containing deprecated section headers not actually overriding
|
||||||
|
configuration options under deprecated section headers.
|
||||||
|
* #835: Add support for the NO_COLOR environment variable. See the documentation for more
|
||||||
|
information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/set-up-backups/#colored-output
|
||||||
|
* #839: Add log sending for the Apprise logging hook, enabled by default. See the documentation for
|
||||||
|
more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook
|
||||||
|
* #839: Document a potentially breaking shell quoting edge case within error hooks:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks
|
||||||
|
* #840: When running the "rcreate" action and the repository already exists but with a different
|
||||||
|
encryption mode than requested, error.
|
||||||
|
* Switch from Drone to Gitea Actions for continuous integration.
|
||||||
|
* Rename scripts/run-end-to-end-dev-tests to scripts/run-end-to-end-tests and use it in both dev
|
||||||
|
and CI for better dev-CI parity.
|
||||||
|
* Clarify documentation about restoring a database: borgmatic does not create the database upon
|
||||||
|
restore.
|
||||||
|
|
||||||
|
1.8.8
|
||||||
|
* #370: For the PostgreSQL hook, pass the "PGSSLMODE" environment variable through to Borg when the
|
||||||
|
database's configuration omits the "ssl_mode" option.
|
||||||
|
* #818: Allow the "--repository" flag to match across multiple configuration files.
|
||||||
|
* #820: Fix broken repository detection in the "rcreate" action with Borg 1.4. The issue did not
|
||||||
|
occur with other versions of Borg.
|
||||||
|
* #822: Fix broken escaping logic in the PostgreSQL hook's "pg_dump_command" option.
|
||||||
|
* SECURITY: Prevent additional shell injection attacks within the PostgreSQL hook.
|
||||||
|
|
||||||
|
1.8.7
|
||||||
|
* #736: Store included configuration files within each backup archive in support of the "config
|
||||||
|
bootstrap" action. Previously, only top-level configuration files were stored.
|
||||||
|
* #798: Elevate specific Borg warnings to errors or squash errors to
|
||||||
|
* warnings. See the documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/customize-warnings-and-errors/
|
||||||
|
* #810: SECURITY: Prevent shell injection attacks within the PostgreSQL hook, the MongoDB hook, the
|
||||||
|
SQLite hook, the "borgmatic borg" action, and command hook variable/constant interpolation.
|
||||||
|
* #814: Fix a traceback when providing an invalid "--override" value for a list option.
|
||||||
|
|
||||||
|
1.8.6
|
||||||
|
* #767: Add an "--ssh-command" flag to the "config bootstrap" action for setting a custom SSH
|
||||||
|
command, as no configuration is available (including the "ssh_command" option) until
|
||||||
|
bootstrapping completes.
|
||||||
|
* #794: Fix a traceback when the "repositories" option contains both strings and key/value pairs.
|
||||||
|
* #800: Add configured repository labels to the JSON output for all actions.
|
||||||
|
* #802: The "check --force" flag now runs checks even if "check" is in "skip_actions".
|
||||||
|
* #804: Validate the configured action names in the "skip_actions" option.
|
||||||
|
* #807: Stream SQLite databases directly to Borg instead of dumping to an intermediate file.
|
||||||
|
* When logging commands that borgmatic executes, log the environment variables that
|
||||||
|
borgmatic sets for those commands. (But don't log their values, since they often contain
|
||||||
|
passwords.)
|
||||||
|
|
||||||
|
1.8.5
|
||||||
|
* #701: Add a "skip_actions" option to skip running particular actions, handy for append-only or
|
||||||
|
checkless configurations. See the documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/set-up-backups/#skipping-actions
|
||||||
|
* #701: Deprecate the "disabled" value for the "checks" option in favor of the new "skip_actions"
|
||||||
|
option.
|
||||||
|
* #745: Constants now apply to included configuration, not just the file doing the includes. As a
|
||||||
|
side effect of this change, constants no longer apply to option names and only substitute into
|
||||||
|
configuration values.
|
||||||
|
* #779: Add a "--match-archives" flag to the "check" action for selecting the archives to check,
|
||||||
|
overriding the existing "archive_name_format" and "match_archives" options in configuration.
|
||||||
|
* #779: Only parse "--override" values as complex data types when they're for options of those
|
||||||
|
types.
|
||||||
|
* #782: Fix environment variable interpolation within configured repository paths.
|
||||||
|
* #782: Add configuration constant overriding via the existing "--override" flag.
|
||||||
|
* #783: Upgrade ruamel.yaml dependency to support version 0.18.x.
|
||||||
|
* #784: Drop support for Python 3.7, which has been end-of-lifed.
|
||||||
|
|
||||||
|
1.8.4
|
||||||
|
* #715: Add a monitoring hook for sending backup status to a variety of monitoring services via the
|
||||||
|
Apprise library. See the documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook
|
||||||
|
* #748: When an archive filter causes no matching archives for the "rlist" or "info"
|
||||||
|
actions, warn the user and suggest how to remove the filter.
|
||||||
|
* #768: Fix a traceback when an invalid command-line flag or action is used.
|
||||||
|
* #771: Fix normalization of deprecated sections ("location:", "storage:", "hooks:", etc.) to
|
||||||
|
support empty sections without erroring.
|
||||||
|
* #774: Disallow the "--dry-run" flag with the "borg" action, as borgmatic can't guarantee the Borg
|
||||||
|
command won't have side effects.
|
||||||
|
|
||||||
|
1.8.3
|
||||||
|
* #665: BREAKING: Simplify logging logic as follows: Syslog verbosity is now disabled by
|
||||||
|
default, but setting the "--syslog-verbosity" flag enables it regardless of whether you're at an
|
||||||
|
interactive console. Additionally, "--log-file-verbosity" and "--monitoring-verbosity" now
|
||||||
|
default to 1 (info about steps borgmatic is taking) instead of 0. And both syslog logging and
|
||||||
|
file logging can be enabled simultaneously.
|
||||||
|
* #743: Add a monitoring hook for sending backup status and logs to Grafana Loki. See the
|
||||||
|
documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook
|
||||||
|
* #753: When "archive_name_format" is not set, filter archives using the default archive name
|
||||||
|
format.
|
||||||
|
* #754: Fix error handling to log command output as one record per line instead of truncating
|
||||||
|
too-long output and swallowing the end of some Borg error messages.
|
||||||
|
* #757: Update documentation so "sudo borgmatic" works for pipx borgmatic installations.
|
||||||
|
* #761: Fix for borgmatic not stopping Borg immediately when the user presses ctrl-C.
|
||||||
|
* Update documentation to recommend installing/upgrading borgmatic with pipx instead of pip. See the
|
||||||
|
documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/set-up-backups/#installation
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-borgmatic
|
||||||
|
|
||||||
1.8.2
|
1.8.2
|
||||||
* #345: Add "key export" action to export a copy of the repository key for safekeeping in case
|
* #345: Add "key export" action to export a copy of the repository key for safekeeping in case
|
||||||
the original goes missing or gets damaged.
|
the original goes missing or gets damaged.
|
||||||
|
@ -38,10 +173,10 @@
|
||||||
"check --repair".
|
"check --repair".
|
||||||
* When merging two configuration files, error gracefully if the two files do not adhere to the same
|
* When merging two configuration files, error gracefully if the two files do not adhere to the same
|
||||||
format.
|
format.
|
||||||
* #721: Remove configuration sections ("location:", "storage:", "hooks:" etc.), while still keeping
|
* #721: Remove configuration sections ("location:", "storage:", "hooks:", etc.), while still
|
||||||
deprecated support for them. Now, all options are at the same level, and you don't need to worry
|
keeping deprecated support for them. Now, all options are at the same level, and you don't need
|
||||||
about commenting/uncommenting section headers when you change an option (if you remove your
|
to worry about commenting/uncommenting section headers when you change an option (if you remove
|
||||||
sections first).
|
your sections first).
|
||||||
* #721: BREAKING: The retention prefix and the consistency prefix can no longer have different
|
* #721: BREAKING: The retention prefix and the consistency prefix can no longer have different
|
||||||
values (unless one is not set).
|
values (unless one is not set).
|
||||||
* #721: BREAKING: The storage umask and the hooks umask can no longer have different values (unless
|
* #721: BREAKING: The storage umask and the hooks umask can no longer have different values (unless
|
||||||
|
|
30
README.md
30
README.md
|
@ -48,24 +48,27 @@ postgresql_databases:
|
||||||
- name: users
|
- name: users
|
||||||
|
|
||||||
# Third-party services to notify you if backups aren't happening.
|
# Third-party services to notify you if backups aren't happening.
|
||||||
healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
|
healthchecks:
|
||||||
|
ping_url: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
|
||||||
```
|
```
|
||||||
|
|
||||||
borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
|
borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
|
||||||
|
|
||||||
## Integrations
|
## Integrations
|
||||||
|
|
||||||
<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>
|
<a href="https://grafana.com/oss/loki/"><img src="docs/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
|
<a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
|
<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||||
|
|
||||||
|
|
||||||
## Getting started
|
## Getting started
|
||||||
|
@ -151,6 +154,3 @@ general, contributions are very welcome. We don't bite!
|
||||||
Also, please check out the [borgmatic development
|
Also, please check out the [borgmatic development
|
||||||
how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
|
how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
|
||||||
info on cloning source code, running tests, etc.
|
info on cloning source code, running tests, etc.
|
||||||
|
|
||||||
<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/main)</a>
|
|
||||||
|
|
||||||
|
|
|
@ -1,12 +1,575 @@
|
||||||
|
import datetime
|
||||||
|
import hashlib
|
||||||
|
import itertools
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import random
|
||||||
|
|
||||||
import borgmatic.borg.check
|
import borgmatic.borg.check
|
||||||
|
import borgmatic.borg.create
|
||||||
|
import borgmatic.borg.environment
|
||||||
|
import borgmatic.borg.extract
|
||||||
|
import borgmatic.borg.list
|
||||||
|
import borgmatic.borg.rlist
|
||||||
|
import borgmatic.borg.state
|
||||||
import borgmatic.config.validate
|
import borgmatic.config.validate
|
||||||
|
import borgmatic.execute
|
||||||
import borgmatic.hooks.command
|
import borgmatic.hooks.command
|
||||||
|
|
||||||
|
DEFAULT_CHECKS = (
|
||||||
|
{'name': 'repository', 'frequency': '1 month'},
|
||||||
|
{'name': 'archives', 'frequency': '1 month'},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_checks(config, only_checks=None):
|
||||||
|
'''
|
||||||
|
Given a configuration dict with a "checks" sequence of dicts and an optional list of override
|
||||||
|
checks, return a tuple of named checks to run.
|
||||||
|
|
||||||
|
For example, given a config of:
|
||||||
|
|
||||||
|
{'checks': ({'name': 'repository'}, {'name': 'archives'})}
|
||||||
|
|
||||||
|
This will be returned as:
|
||||||
|
|
||||||
|
('repository', 'archives')
|
||||||
|
|
||||||
|
If no "checks" option is present in the config, return the DEFAULT_CHECKS. If a checks value
|
||||||
|
has a name of "disabled", return an empty tuple, meaning that no checks should be run.
|
||||||
|
'''
|
||||||
|
checks = only_checks or tuple(
|
||||||
|
check_config['name'] for check_config in (config.get('checks', None) or DEFAULT_CHECKS)
|
||||||
|
)
|
||||||
|
checks = tuple(check.lower() for check in checks)
|
||||||
|
|
||||||
|
if 'disabled' in checks:
|
||||||
|
logger.warning(
|
||||||
|
'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead'
|
||||||
|
)
|
||||||
|
if len(checks) > 1:
|
||||||
|
logger.warning(
|
||||||
|
'Multiple checks are configured, but one of them is "disabled"; not running any checks'
|
||||||
|
)
|
||||||
|
return ()
|
||||||
|
|
||||||
|
return checks
|
||||||
|
|
||||||
|
|
||||||
|
def parse_frequency(frequency):
|
||||||
|
'''
|
||||||
|
Given a frequency string with a number and a unit of time, return a corresponding
|
||||||
|
datetime.timedelta instance or None if the frequency is None or "always".
|
||||||
|
|
||||||
|
For instance, given "3 weeks", return datetime.timedelta(weeks=3)
|
||||||
|
|
||||||
|
Raise ValueError if the given frequency cannot be parsed.
|
||||||
|
'''
|
||||||
|
if not frequency:
|
||||||
|
return None
|
||||||
|
|
||||||
|
frequency = frequency.strip().lower()
|
||||||
|
|
||||||
|
if frequency == 'always':
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
number, time_unit = frequency.split(' ')
|
||||||
|
number = int(number)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
|
||||||
|
|
||||||
|
if not time_unit.endswith('s'):
|
||||||
|
time_unit += 's'
|
||||||
|
|
||||||
|
if time_unit == 'months':
|
||||||
|
number *= 30
|
||||||
|
time_unit = 'days'
|
||||||
|
elif time_unit == 'years':
|
||||||
|
number *= 365
|
||||||
|
time_unit = 'days'
|
||||||
|
|
||||||
|
try:
|
||||||
|
return datetime.timedelta(**{time_unit: number})
|
||||||
|
except TypeError:
|
||||||
|
raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
|
||||||
|
|
||||||
|
|
||||||
|
def filter_checks_on_frequency(
|
||||||
|
config,
|
||||||
|
borg_repository_id,
|
||||||
|
checks,
|
||||||
|
force,
|
||||||
|
archives_check_id=None,
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Given a configuration dict with a "checks" sequence of dicts, a Borg repository ID, a sequence
|
||||||
|
of checks, whether to force checks to run, and an ID for the archives check potentially being
|
||||||
|
run (if any), filter down those checks based on the configured "frequency" for each check as
|
||||||
|
compared to its check time file.
|
||||||
|
|
||||||
|
In other words, a check whose check time file's timestamp is too new (based on the configured
|
||||||
|
frequency) will get cut from the returned sequence of checks. Example:
|
||||||
|
|
||||||
|
config = {
|
||||||
|
'checks': [
|
||||||
|
{
|
||||||
|
'name': 'archives',
|
||||||
|
'frequency': '2 weeks',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
When this function is called with that config and "archives" in checks, "archives" will get
|
||||||
|
filtered out of the returned result if its check time file is newer than 2 weeks old, indicating
|
||||||
|
that it's not yet time to run that check again.
|
||||||
|
|
||||||
|
Raise ValueError if a frequency cannot be parsed.
|
||||||
|
'''
|
||||||
|
if not checks:
|
||||||
|
return checks
|
||||||
|
|
||||||
|
filtered_checks = list(checks)
|
||||||
|
|
||||||
|
if force:
|
||||||
|
return tuple(filtered_checks)
|
||||||
|
|
||||||
|
for check_config in config.get('checks', DEFAULT_CHECKS):
|
||||||
|
check = check_config['name']
|
||||||
|
if checks and check not in checks:
|
||||||
|
continue
|
||||||
|
|
||||||
|
frequency_delta = parse_frequency(check_config.get('frequency'))
|
||||||
|
if not frequency_delta:
|
||||||
|
continue
|
||||||
|
|
||||||
|
check_time = probe_for_check_time(config, borg_repository_id, check, archives_check_id)
|
||||||
|
if not check_time:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# If we've not yet reached the time when the frequency dictates we're ready for another
|
||||||
|
# check, skip this check.
|
||||||
|
if datetime.datetime.now() < check_time + frequency_delta:
|
||||||
|
remaining = check_time + frequency_delta - datetime.datetime.now()
|
||||||
|
logger.info(
|
||||||
|
f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
|
||||||
|
)
|
||||||
|
filtered_checks.remove(check)
|
||||||
|
|
||||||
|
return tuple(filtered_checks)
|
||||||
|
|
||||||
|
|
||||||
|
def make_archives_check_id(archive_filter_flags):
|
||||||
|
'''
|
||||||
|
Given a sequence of flags to filter archives, return a unique hash corresponding to those
|
||||||
|
particular flags. If there are no flags, return None.
|
||||||
|
'''
|
||||||
|
if not archive_filter_flags:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def make_check_time_path(config, borg_repository_id, check_type, archives_check_id=None):
|
||||||
|
'''
|
||||||
|
Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
|
||||||
|
"archives", etc.), and a unique hash of the archives filter flags, return a path for recording
|
||||||
|
that check's time (the time of that check last occurring).
|
||||||
|
'''
|
||||||
|
borgmatic_source_directory = os.path.expanduser(
|
||||||
|
config.get(
|
||||||
|
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if check_type in ('archives', 'data'):
|
||||||
|
return os.path.join(
|
||||||
|
borgmatic_source_directory,
|
||||||
|
'checks',
|
||||||
|
borg_repository_id,
|
||||||
|
check_type,
|
||||||
|
archives_check_id if archives_check_id else 'all',
|
||||||
|
)
|
||||||
|
|
||||||
|
return os.path.join(
|
||||||
|
borgmatic_source_directory,
|
||||||
|
'checks',
|
||||||
|
borg_repository_id,
|
||||||
|
check_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_check_time(path): # pragma: no cover
|
||||||
|
'''
|
||||||
|
Record a check time of now as the modification time of the given path.
|
||||||
|
'''
|
||||||
|
logger.debug(f'Writing check time at {path}')
|
||||||
|
|
||||||
|
os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
|
||||||
|
pathlib.Path(path, mode=0o600).touch()
|
||||||
|
|
||||||
|
|
||||||
|
def read_check_time(path):
|
||||||
|
'''
|
||||||
|
Return the check time based on the modification time of the given path. Return None if the path
|
||||||
|
doesn't exist.
|
||||||
|
'''
|
||||||
|
logger.debug(f'Reading check time from {path}')
|
||||||
|
|
||||||
|
try:
|
||||||
|
return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
|
||||||
|
'''
|
||||||
|
Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
|
||||||
|
"archives", etc.), and a unique hash of the archives filter flags, return a the corresponding
|
||||||
|
check time or None if such a check time does not exist.
|
||||||
|
|
||||||
|
When the check type is "archives" or "data", this function probes two different paths to find
|
||||||
|
the check time, e.g.:
|
||||||
|
|
||||||
|
~/.borgmatic/checks/1234567890/archives/9876543210
|
||||||
|
~/.borgmatic/checks/1234567890/archives/all
|
||||||
|
|
||||||
|
... and returns the maximum modification time of the files found (if any). The first path
|
||||||
|
represents a more specific archives check time (a check on a subset of archives), and the second
|
||||||
|
is a fallback to the last "all" archives check.
|
||||||
|
|
||||||
|
For other check types, this function reads from a single check time path, e.g.:
|
||||||
|
|
||||||
|
~/.borgmatic/checks/1234567890/repository
|
||||||
|
'''
|
||||||
|
check_times = (
|
||||||
|
read_check_time(group[0])
|
||||||
|
for group in itertools.groupby(
|
||||||
|
(
|
||||||
|
make_check_time_path(config, borg_repository_id, check, archives_check_id),
|
||||||
|
make_check_time_path(config, borg_repository_id, check),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
return max(check_time for check_time in check_times if check_time)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade_check_times(config, borg_repository_id):
|
||||||
|
'''
|
||||||
|
Given a configuration dict and a Borg repository ID, upgrade any corresponding check times on
|
||||||
|
disk from old-style paths to new-style paths.
|
||||||
|
|
||||||
|
Currently, the only upgrade performed is renaming an archive or data check path that looks like:
|
||||||
|
|
||||||
|
~/.borgmatic/checks/1234567890/archives
|
||||||
|
|
||||||
|
to:
|
||||||
|
|
||||||
|
~/.borgmatic/checks/1234567890/archives/all
|
||||||
|
'''
|
||||||
|
for check_type in ('archives', 'data'):
|
||||||
|
new_path = make_check_time_path(config, borg_repository_id, check_type, 'all')
|
||||||
|
old_path = os.path.dirname(new_path)
|
||||||
|
temporary_path = f'{old_path}.temp'
|
||||||
|
|
||||||
|
if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.debug(f'Upgrading archives check time from {old_path} to {new_path}')
|
||||||
|
|
||||||
|
try:
|
||||||
|
os.rename(old_path, temporary_path)
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
os.mkdir(old_path)
|
||||||
|
os.rename(temporary_path, new_path)
|
||||||
|
|
||||||
|
|
||||||
|
def collect_spot_check_source_paths(
|
||||||
|
repository, config, local_borg_version, global_arguments, local_path, remote_path
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Given a repository configuration dict, a configuration dict, the local Borg version, global
|
||||||
|
arguments as an argparse.Namespace instance, the local Borg path, and the remote Borg path,
|
||||||
|
collect the source paths that Borg would use in an actual create (but only include files and
|
||||||
|
symlinks).
|
||||||
|
'''
|
||||||
|
stream_processes = any(
|
||||||
|
borgmatic.hooks.dispatch.call_hooks(
|
||||||
|
'use_streaming',
|
||||||
|
config,
|
||||||
|
repository['path'],
|
||||||
|
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
|
||||||
|
).values()
|
||||||
|
)
|
||||||
|
|
||||||
|
(create_flags, create_positional_arguments, pattern_file, exclude_file) = (
|
||||||
|
borgmatic.borg.create.make_base_create_command(
|
||||||
|
dry_run=True,
|
||||||
|
repository_path=repository['path'],
|
||||||
|
config=config,
|
||||||
|
config_paths=(),
|
||||||
|
local_borg_version=local_borg_version,
|
||||||
|
global_arguments=global_arguments,
|
||||||
|
borgmatic_source_directories=(),
|
||||||
|
local_path=local_path,
|
||||||
|
remote_path=remote_path,
|
||||||
|
list_files=True,
|
||||||
|
stream_processes=stream_processes,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
borg_environment = borgmatic.borg.environment.make_environment(config)
|
||||||
|
|
||||||
|
try:
|
||||||
|
working_directory = os.path.expanduser(config.get('working_directory'))
|
||||||
|
except TypeError:
|
||||||
|
working_directory = None
|
||||||
|
|
||||||
|
paths_output = borgmatic.execute.execute_command_and_capture_output(
|
||||||
|
create_flags + create_positional_arguments,
|
||||||
|
capture_stderr=True,
|
||||||
|
working_directory=working_directory,
|
||||||
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
|
)
|
||||||
|
|
||||||
|
paths = tuple(
|
||||||
|
path_line.split(' ', 1)[1]
|
||||||
|
for path_line in paths_output.split('\n')
|
||||||
|
if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
|
||||||
|
)
|
||||||
|
|
||||||
|
return tuple(path for path in paths if os.path.isfile(path) or os.path.islink(path))
|
||||||
|
|
||||||
|
|
||||||
|
BORG_DIRECTORY_FILE_TYPE = 'd'
|
||||||
|
|
||||||
|
|
||||||
|
def collect_spot_check_archive_paths(
|
||||||
|
repository, archive, config, local_borg_version, global_arguments, local_path, remote_path
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Given a repository configuration dict, the name of the latest archive, a configuration dict, the
|
||||||
|
local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, and
|
||||||
|
the remote Borg path, collect the paths from the given archive (but only include files and
|
||||||
|
symlinks).
|
||||||
|
'''
|
||||||
|
borgmatic_source_directory = os.path.expanduser(
|
||||||
|
config.get(
|
||||||
|
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return tuple(
|
||||||
|
path
|
||||||
|
for line in borgmatic.borg.list.capture_archive_listing(
|
||||||
|
repository['path'],
|
||||||
|
archive,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
path_format='{type} /{path}{NL}', # noqa: FS003
|
||||||
|
local_path=local_path,
|
||||||
|
remote_path=remote_path,
|
||||||
|
)
|
||||||
|
for (file_type, path) in (line.split(' ', 1),)
|
||||||
|
if file_type != BORG_DIRECTORY_FILE_TYPE
|
||||||
|
if pathlib.Path(borgmatic_source_directory) not in pathlib.Path(path).parents
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def compare_spot_check_hashes(
|
||||||
|
repository,
|
||||||
|
archive,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
log_label,
|
||||||
|
source_paths,
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Given a repository configuration dict, the name of the latest archive, a configuration dict, the
|
||||||
|
local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the
|
||||||
|
remote Borg path, a log label, and spot check source paths, compare the hashes for a sampling of
|
||||||
|
the source paths with hashes from corresponding paths in the given archive. Return a sequence of
|
||||||
|
the paths that fail that hash comparison.
|
||||||
|
'''
|
||||||
|
# Based on the configured sample percentage, come up with a list of random sample files from the
|
||||||
|
# source directories.
|
||||||
|
spot_check_config = next(check for check in config['checks'] if check['name'] == 'spot')
|
||||||
|
sample_count = max(
|
||||||
|
int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
|
||||||
|
)
|
||||||
|
source_sample_paths = tuple(random.sample(source_paths, sample_count))
|
||||||
|
existing_source_sample_paths = {
|
||||||
|
source_path for source_path in source_sample_paths if os.path.exists(source_path)
|
||||||
|
}
|
||||||
|
logger.debug(
|
||||||
|
f'{log_label}: Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Hash each file in the sample paths (if it exists).
|
||||||
|
hash_output = borgmatic.execute.execute_command_and_capture_output(
|
||||||
|
(spot_check_config.get('xxh64sum_command', 'xxh64sum'),)
|
||||||
|
+ tuple(path for path in source_sample_paths if path in existing_source_sample_paths)
|
||||||
|
)
|
||||||
|
|
||||||
|
source_hashes = dict(
|
||||||
|
(reversed(line.split(' ', 1)) for line in hash_output.splitlines()),
|
||||||
|
**{path: '' for path in source_sample_paths if path not in existing_source_sample_paths},
|
||||||
|
)
|
||||||
|
|
||||||
|
archive_hashes = dict(
|
||||||
|
reversed(line.split(' ', 1))
|
||||||
|
for line in borgmatic.borg.list.capture_archive_listing(
|
||||||
|
repository['path'],
|
||||||
|
archive,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
list_paths=source_sample_paths,
|
||||||
|
path_format='{xxh64} /{path}{NL}', # noqa: FS003
|
||||||
|
local_path=local_path,
|
||||||
|
remote_path=remote_path,
|
||||||
|
)
|
||||||
|
if line
|
||||||
|
)
|
||||||
|
|
||||||
|
# Compare the source hashes with the archive hashes to see how many match.
|
||||||
|
failing_paths = []
|
||||||
|
|
||||||
|
for path, source_hash in source_hashes.items():
|
||||||
|
archive_hash = archive_hashes.get(path)
|
||||||
|
|
||||||
|
if archive_hash is not None and archive_hash == source_hash:
|
||||||
|
continue
|
||||||
|
|
||||||
|
failing_paths.append(path)
|
||||||
|
|
||||||
|
return tuple(failing_paths)
|
||||||
|
|
||||||
|
|
||||||
|
def spot_check(
|
||||||
|
repository,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Given a repository dict, a loaded configuration dict, the local Borg version, global arguments
|
||||||
|
as an argparse.Namespace instance, the local Borg path, and the remote Borg path, perform a spot
|
||||||
|
check for the latest archive in the given repository.
|
||||||
|
|
||||||
|
A spot check compares file counts and also the hashes for a random sampling of source files on
|
||||||
|
disk to those stored in the latest archive. If any differences are beyond configured tolerances,
|
||||||
|
then the check fails.
|
||||||
|
'''
|
||||||
|
log_label = f'{repository.get("label", repository["path"])}'
|
||||||
|
logger.debug(f'{log_label}: Running spot check')
|
||||||
|
|
||||||
|
try:
|
||||||
|
spot_check_config = next(
|
||||||
|
check for check in config.get('checks', ()) if check.get('name') == 'spot'
|
||||||
|
)
|
||||||
|
except StopIteration:
|
||||||
|
raise ValueError('Cannot run spot check because it is unconfigured')
|
||||||
|
|
||||||
|
if spot_check_config['data_tolerance_percentage'] > spot_check_config['data_sample_percentage']:
|
||||||
|
raise ValueError(
|
||||||
|
'The data_tolerance_percentage must be less than or equal to the data_sample_percentage'
|
||||||
|
)
|
||||||
|
|
||||||
|
source_paths = collect_spot_check_source_paths(
|
||||||
|
repository,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
)
|
||||||
|
logger.debug(f'{log_label}: {len(source_paths)} total source paths for spot check')
|
||||||
|
|
||||||
|
archive = borgmatic.borg.rlist.resolve_archive_name(
|
||||||
|
repository['path'],
|
||||||
|
'latest',
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
)
|
||||||
|
logger.debug(f'{log_label}: Using archive {archive} for spot check')
|
||||||
|
|
||||||
|
archive_paths = collect_spot_check_archive_paths(
|
||||||
|
repository,
|
||||||
|
archive,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
)
|
||||||
|
logger.debug(f'{log_label}: {len(archive_paths)} total archive paths for spot check')
|
||||||
|
|
||||||
|
# Calculate the percentage delta between the source paths count and the archive paths count, and
|
||||||
|
# compare that delta to the configured count tolerance percentage.
|
||||||
|
count_delta_percentage = abs(len(source_paths) - len(archive_paths)) / len(source_paths) * 100
|
||||||
|
|
||||||
|
if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
|
||||||
|
logger.debug(
|
||||||
|
f'{log_label}: Paths in source paths but not latest archive: {", ".join(set(source_paths) - set(archive_paths)) or "none"}'
|
||||||
|
)
|
||||||
|
logger.debug(
|
||||||
|
f'{log_label}: Paths in latest archive but not source paths: {", ".join(set(archive_paths) - set(source_paths)) or "none"}'
|
||||||
|
)
|
||||||
|
raise ValueError(
|
||||||
|
f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'
|
||||||
|
)
|
||||||
|
|
||||||
|
failing_paths = compare_spot_check_hashes(
|
||||||
|
repository,
|
||||||
|
archive,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
log_label,
|
||||||
|
source_paths,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Error if the percentage of failing hashes exceeds the configured tolerance percentage.
|
||||||
|
logger.debug(f'{log_label}: {len(failing_paths)} non-matching spot check hashes')
|
||||||
|
data_tolerance_percentage = spot_check_config['data_tolerance_percentage']
|
||||||
|
failing_percentage = (len(failing_paths) / len(source_paths)) * 100
|
||||||
|
|
||||||
|
if failing_percentage > data_tolerance_percentage:
|
||||||
|
logger.debug(
|
||||||
|
f'{log_label}: Source paths with data not matching the latest archive: {", ".join(failing_paths)}'
|
||||||
|
)
|
||||||
|
raise ValueError(
|
||||||
|
f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)'
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f'{log_label}: Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def run_check(
|
def run_check(
|
||||||
config_filename,
|
config_filename,
|
||||||
repository,
|
repository,
|
||||||
|
@ -20,6 +583,8 @@ def run_check(
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Run the "check" action for the given repository.
|
Run the "check" action for the given repository.
|
||||||
|
|
||||||
|
Raise ValueError if the Borg repository ID cannot be determined.
|
||||||
'''
|
'''
|
||||||
if check_arguments.repository and not borgmatic.config.validate.repositories_match(
|
if check_arguments.repository and not borgmatic.config.validate.repositories_match(
|
||||||
repository, check_arguments.repository
|
repository, check_arguments.repository
|
||||||
|
@ -34,19 +599,69 @@ def run_check(
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
**hook_context,
|
**hook_context,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks')
|
logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks')
|
||||||
borgmatic.borg.check.check_archives(
|
repository_id = borgmatic.borg.check.get_repository_id(
|
||||||
repository['path'],
|
repository['path'],
|
||||||
config,
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
local_path=local_path,
|
local_path=local_path,
|
||||||
remote_path=remote_path,
|
remote_path=remote_path,
|
||||||
progress=check_arguments.progress,
|
|
||||||
repair=check_arguments.repair,
|
|
||||||
only_checks=check_arguments.only,
|
|
||||||
force=check_arguments.force,
|
|
||||||
)
|
)
|
||||||
|
upgrade_check_times(config, repository_id)
|
||||||
|
configured_checks = parse_checks(config, check_arguments.only_checks)
|
||||||
|
archive_filter_flags = borgmatic.borg.check.make_archive_filter_flags(
|
||||||
|
local_borg_version, config, configured_checks, check_arguments
|
||||||
|
)
|
||||||
|
archives_check_id = make_archives_check_id(archive_filter_flags)
|
||||||
|
checks = filter_checks_on_frequency(
|
||||||
|
config,
|
||||||
|
repository_id,
|
||||||
|
configured_checks,
|
||||||
|
check_arguments.force,
|
||||||
|
archives_check_id,
|
||||||
|
)
|
||||||
|
borg_specific_checks = set(checks).intersection({'repository', 'archives', 'data'})
|
||||||
|
|
||||||
|
if borg_specific_checks:
|
||||||
|
borgmatic.borg.check.check_archives(
|
||||||
|
repository['path'],
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
check_arguments,
|
||||||
|
global_arguments,
|
||||||
|
borg_specific_checks,
|
||||||
|
archive_filter_flags,
|
||||||
|
local_path=local_path,
|
||||||
|
remote_path=remote_path,
|
||||||
|
)
|
||||||
|
for check in borg_specific_checks:
|
||||||
|
write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
|
||||||
|
|
||||||
|
if 'extract' in checks:
|
||||||
|
borgmatic.borg.extract.extract_last_archive_dry_run(
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
repository['path'],
|
||||||
|
config.get('lock_wait'),
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
)
|
||||||
|
write_check_time(make_check_time_path(config, repository_id, 'extract'))
|
||||||
|
|
||||||
|
if 'spot' in checks:
|
||||||
|
spot_check(
|
||||||
|
repository,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
)
|
||||||
|
write_check_time(make_check_time_path(config, repository_id, 'spot'))
|
||||||
|
|
||||||
borgmatic.hooks.command.execute_hook(
|
borgmatic.hooks.command.execute_hook(
|
||||||
config.get('after_check'),
|
config.get('after_check'),
|
||||||
config.get('umask'),
|
config.get('umask'),
|
||||||
|
|
|
@ -13,14 +13,11 @@ logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
|
def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
|
||||||
'''
|
'''
|
||||||
Given:
|
Given the bootstrap arguments as an argparse.Namespace (containing the repository and archive
|
||||||
The bootstrap arguments, which include the repository and archive name, borgmatic source directory,
|
name, borgmatic source directory, destination directory, and whether to strip components), the
|
||||||
destination directory, and whether to strip components.
|
global arguments as an argparse.Namespace (containing the dry run flag and the local borg
|
||||||
The global arguments, which include the dry run flag
|
version), return the config paths from the manifest.json file in the borgmatic source directory
|
||||||
and the local borg version,
|
after extracting it from the repository.
|
||||||
Return:
|
|
||||||
The config paths from the manifest.json file in the borgmatic source directory after extracting it from the
|
|
||||||
repository.
|
|
||||||
|
|
||||||
Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
|
Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
|
||||||
expected configuration path data.
|
expected configuration path data.
|
||||||
|
@ -31,24 +28,26 @@ def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
|
||||||
borgmatic_manifest_path = os.path.expanduser(
|
borgmatic_manifest_path = os.path.expanduser(
|
||||||
os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
|
os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
|
||||||
)
|
)
|
||||||
|
config = {'ssh_command': bootstrap_arguments.ssh_command}
|
||||||
|
|
||||||
extract_process = borgmatic.borg.extract.extract_archive(
|
extract_process = borgmatic.borg.extract.extract_archive(
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
bootstrap_arguments.repository,
|
bootstrap_arguments.repository,
|
||||||
borgmatic.borg.rlist.resolve_archive_name(
|
borgmatic.borg.rlist.resolve_archive_name(
|
||||||
bootstrap_arguments.repository,
|
bootstrap_arguments.repository,
|
||||||
bootstrap_arguments.archive,
|
bootstrap_arguments.archive,
|
||||||
{},
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
),
|
),
|
||||||
[borgmatic_manifest_path],
|
[borgmatic_manifest_path],
|
||||||
{},
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
extract_to_stdout=True,
|
extract_to_stdout=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
manifest_json = extract_process.stdout.read()
|
manifest_json = extract_process.stdout.read()
|
||||||
|
|
||||||
if not manifest_json:
|
if not manifest_json:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
'Cannot read configuration paths from archive due to missing bootstrap manifest'
|
'Cannot read configuration paths from archive due to missing bootstrap manifest'
|
||||||
|
@ -79,6 +78,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
|
||||||
manifest_config_paths = get_config_paths(
|
manifest_config_paths = get_config_paths(
|
||||||
bootstrap_arguments, global_arguments, local_borg_version
|
bootstrap_arguments, global_arguments, local_borg_version
|
||||||
)
|
)
|
||||||
|
config = {'ssh_command': bootstrap_arguments.ssh_command}
|
||||||
|
|
||||||
logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
|
logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
|
||||||
|
|
||||||
|
@ -88,12 +88,12 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
|
||||||
borgmatic.borg.rlist.resolve_archive_name(
|
borgmatic.borg.rlist.resolve_archive_name(
|
||||||
bootstrap_arguments.repository,
|
bootstrap_arguments.repository,
|
||||||
bootstrap_arguments.archive,
|
bootstrap_arguments.archive,
|
||||||
{},
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
),
|
),
|
||||||
[config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
|
[config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
|
||||||
{},
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
extract_to_stdout=False,
|
extract_to_stdout=False,
|
||||||
|
|
|
@ -1,12 +1,9 @@
|
||||||
|
import importlib.metadata
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
try:
|
import borgmatic.actions.json
|
||||||
import importlib_metadata
|
|
||||||
except ModuleNotFoundError: # pragma: nocover
|
|
||||||
import importlib.metadata as importlib_metadata
|
|
||||||
|
|
||||||
import borgmatic.borg.create
|
import borgmatic.borg.create
|
||||||
import borgmatic.borg.state
|
import borgmatic.borg.state
|
||||||
import borgmatic.config.validate
|
import borgmatic.config.validate
|
||||||
|
@ -39,7 +36,7 @@ def create_borgmatic_manifest(config, config_paths, dry_run):
|
||||||
with open(borgmatic_manifest_path, 'w') as config_list_file:
|
with open(borgmatic_manifest_path, 'w') as config_list_file:
|
||||||
json.dump(
|
json.dump(
|
||||||
{
|
{
|
||||||
'borgmatic_version': importlib_metadata.version('borgmatic'),
|
'borgmatic_version': importlib.metadata.version('borgmatic'),
|
||||||
'config_paths': config_paths,
|
'config_paths': config_paths,
|
||||||
},
|
},
|
||||||
config_list_file,
|
config_list_file,
|
||||||
|
@ -50,6 +47,7 @@ def run_create(
|
||||||
config_filename,
|
config_filename,
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
config_paths,
|
||||||
hook_context,
|
hook_context,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
create_arguments,
|
create_arguments,
|
||||||
|
@ -78,22 +76,24 @@ def run_create(
|
||||||
)
|
)
|
||||||
logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}')
|
logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}')
|
||||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||||
'remove_database_dumps',
|
'remove_data_source_dumps',
|
||||||
config,
|
config,
|
||||||
repository['path'],
|
repository['path'],
|
||||||
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
|
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
active_dumps = borgmatic.hooks.dispatch.call_hooks(
|
active_dumps = borgmatic.hooks.dispatch.call_hooks(
|
||||||
'dump_databases',
|
'dump_data_sources',
|
||||||
config,
|
config,
|
||||||
repository['path'],
|
repository['path'],
|
||||||
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
|
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
if config.get('store_config_files', True):
|
if config.get('store_config_files', True):
|
||||||
create_borgmatic_manifest(
|
create_borgmatic_manifest(
|
||||||
config, global_arguments.used_config_paths, global_arguments.dry_run
|
config,
|
||||||
|
config_paths,
|
||||||
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
stream_processes = [process for processes in active_dumps.values() for process in processes]
|
stream_processes = [process for processes in active_dumps.values() for process in processes]
|
||||||
|
|
||||||
|
@ -101,6 +101,7 @@ def run_create(
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
repository['path'],
|
repository['path'],
|
||||||
config,
|
config,
|
||||||
|
config_paths,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
local_path=local_path,
|
local_path=local_path,
|
||||||
|
@ -111,14 +112,14 @@ def run_create(
|
||||||
list_files=create_arguments.list_files,
|
list_files=create_arguments.list_files,
|
||||||
stream_processes=stream_processes,
|
stream_processes=stream_processes,
|
||||||
)
|
)
|
||||||
if json_output: # pragma: nocover
|
if json_output:
|
||||||
yield json.loads(json_output)
|
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||||
|
|
||||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||||
'remove_database_dumps',
|
'remove_data_source_dumps',
|
||||||
config,
|
config,
|
||||||
config_filename,
|
config_filename,
|
||||||
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
|
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
borgmatic.hooks.command.execute_hook(
|
borgmatic.hooks.command.execute_hook(
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import borgmatic.actions.arguments
|
import borgmatic.actions.arguments
|
||||||
|
import borgmatic.actions.json
|
||||||
import borgmatic.borg.info
|
import borgmatic.borg.info
|
||||||
import borgmatic.borg.rlist
|
import borgmatic.borg.rlist
|
||||||
import borgmatic.config.validate
|
import borgmatic.config.validate
|
||||||
|
@ -26,7 +26,7 @@ def run_info(
|
||||||
if info_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
if info_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||||
repository, info_arguments.repository
|
repository, info_arguments.repository
|
||||||
):
|
):
|
||||||
if not info_arguments.json: # pragma: nocover
|
if not info_arguments.json:
|
||||||
logger.answer(
|
logger.answer(
|
||||||
f'{repository.get("label", repository["path"])}: Displaying archive summary information'
|
f'{repository.get("label", repository["path"])}: Displaying archive summary information'
|
||||||
)
|
)
|
||||||
|
@ -48,5 +48,5 @@ def run_info(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
if json_output: # pragma: nocover
|
if json_output:
|
||||||
yield json.loads(json_output)
|
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||||
|
|
|
@ -0,0 +1,30 @@
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_json(borg_json_output, label):
|
||||||
|
'''
|
||||||
|
Given a Borg JSON output string, parse it as JSON into a dict. Inject the given borgmatic
|
||||||
|
repository label into it and return the dict.
|
||||||
|
|
||||||
|
Raise JSONDecodeError if the JSON output cannot be parsed.
|
||||||
|
'''
|
||||||
|
lines = borg_json_output.splitlines()
|
||||||
|
start_line_index = 0
|
||||||
|
|
||||||
|
# Scan forward to find the first line starting with "{" and assume that's where the JSON starts.
|
||||||
|
for line_index, line in enumerate(lines):
|
||||||
|
if line.startswith('{'):
|
||||||
|
start_line_index = line_index
|
||||||
|
break
|
||||||
|
|
||||||
|
json_data = json.loads('\n'.join(lines[start_line_index:]))
|
||||||
|
|
||||||
|
if 'repository' not in json_data:
|
||||||
|
return json_data
|
||||||
|
|
||||||
|
json_data['repository']['label'] = label or ''
|
||||||
|
|
||||||
|
return json_data
|
|
@ -1,7 +1,7 @@
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import borgmatic.actions.arguments
|
import borgmatic.actions.arguments
|
||||||
|
import borgmatic.actions.json
|
||||||
import borgmatic.borg.list
|
import borgmatic.borg.list
|
||||||
import borgmatic.config.validate
|
import borgmatic.config.validate
|
||||||
|
|
||||||
|
@ -25,10 +25,10 @@ def run_list(
|
||||||
if list_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
if list_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||||
repository, list_arguments.repository
|
repository, list_arguments.repository
|
||||||
):
|
):
|
||||||
if not list_arguments.json: # pragma: nocover
|
if not list_arguments.json:
|
||||||
if list_arguments.find_paths:
|
if list_arguments.find_paths: # pragma: no cover
|
||||||
logger.answer(f'{repository.get("label", repository["path"])}: Searching archives')
|
logger.answer(f'{repository.get("label", repository["path"])}: Searching archives')
|
||||||
elif not list_arguments.archive:
|
elif not list_arguments.archive: # pragma: no cover
|
||||||
logger.answer(f'{repository.get("label", repository["path"])}: Listing archives')
|
logger.answer(f'{repository.get("label", repository["path"])}: Listing archives')
|
||||||
|
|
||||||
archive_name = borgmatic.borg.rlist.resolve_archive_name(
|
archive_name = borgmatic.borg.rlist.resolve_archive_name(
|
||||||
|
@ -49,5 +49,5 @@ def run_list(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
if json_output: # pragma: nocover
|
if json_output:
|
||||||
yield json.loads(json_output)
|
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||||
|
|
|
@ -17,27 +17,31 @@ logger = logging.getLogger(__name__)
|
||||||
UNSPECIFIED_HOOK = object()
|
UNSPECIFIED_HOOK = object()
|
||||||
|
|
||||||
|
|
||||||
def get_configured_database(
|
def get_configured_data_source(
|
||||||
config, archive_database_names, hook_name, database_name, configuration_database_name=None
|
config,
|
||||||
|
archive_data_source_names,
|
||||||
|
hook_name,
|
||||||
|
data_source_name,
|
||||||
|
configuration_data_source_name=None,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Find the first database with the given hook name and database name in the configuration dict and
|
Find the first data source with the given hook name and data source name in the configuration
|
||||||
the given archive database names dict (from hook name to database names contained in a
|
dict and the given archive data source names dict (from hook name to data source names contained
|
||||||
particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all database
|
in a particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all data
|
||||||
hooks for the named database. If a configuration database name is given, use that instead of the
|
source hooks for the named data source. If a configuration data source name is given, use that
|
||||||
database name to lookup the database in the given hooks configuration.
|
instead of the data source name to lookup the data source in the given hooks configuration.
|
||||||
|
|
||||||
Return the found database as a tuple of (found hook name, database configuration dict) or (None,
|
Return the found data source as a tuple of (found hook name, data source configuration dict) or
|
||||||
None) if not found.
|
(None, None) if not found.
|
||||||
'''
|
'''
|
||||||
if not configuration_database_name:
|
if not configuration_data_source_name:
|
||||||
configuration_database_name = database_name
|
configuration_data_source_name = data_source_name
|
||||||
|
|
||||||
if hook_name == UNSPECIFIED_HOOK:
|
if hook_name == UNSPECIFIED_HOOK:
|
||||||
hooks_to_search = {
|
hooks_to_search = {
|
||||||
hook_name: value
|
hook_name: value
|
||||||
for (hook_name, value) in config.items()
|
for (hook_name, value) in config.items()
|
||||||
if hook_name in borgmatic.hooks.dump.DATABASE_HOOK_NAMES
|
if hook_name in borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES
|
||||||
}
|
}
|
||||||
else:
|
else:
|
||||||
try:
|
try:
|
||||||
|
@ -47,24 +51,17 @@ def get_configured_database(
|
||||||
|
|
||||||
return next(
|
return next(
|
||||||
(
|
(
|
||||||
(name, hook_database)
|
(name, hook_data_source)
|
||||||
for (name, hook) in hooks_to_search.items()
|
for (name, hook) in hooks_to_search.items()
|
||||||
for hook_database in hook
|
for hook_data_source in hook
|
||||||
if hook_database['name'] == configuration_database_name
|
if hook_data_source['name'] == configuration_data_source_name
|
||||||
and database_name in archive_database_names.get(name, [])
|
and data_source_name in archive_data_source_names.get(name, [])
|
||||||
),
|
),
|
||||||
(None, None),
|
(None, None),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_configured_hook_name_and_database(hooks, database_name):
|
def restore_single_data_source(
|
||||||
'''
|
|
||||||
Find the hook name and first database dict with the given database name in the configured hooks
|
|
||||||
dict. This searches across all database hooks.
|
|
||||||
'''
|
|
||||||
|
|
||||||
|
|
||||||
def restore_single_database(
|
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
|
@ -73,27 +70,27 @@ def restore_single_database(
|
||||||
remote_path,
|
remote_path,
|
||||||
archive_name,
|
archive_name,
|
||||||
hook_name,
|
hook_name,
|
||||||
database,
|
data_source,
|
||||||
connection_params,
|
connection_params,
|
||||||
): # pragma: no cover
|
): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Given (among other things) an archive name, a database hook name, the hostname, port,
|
Given (among other things) an archive name, a data source hook name, the hostname, port,
|
||||||
username/password as connection params, and a configured database configuration dict, restore
|
username/password as connection params, and a configured data source configuration dict, restore
|
||||||
that database from the archive.
|
that data source from the archive.
|
||||||
'''
|
'''
|
||||||
logger.info(
|
logger.info(
|
||||||
f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}'
|
f'{repository.get("label", repository["path"])}: Restoring data source {data_source["name"]}'
|
||||||
)
|
)
|
||||||
|
|
||||||
dump_pattern = borgmatic.hooks.dispatch.call_hooks(
|
dump_pattern = borgmatic.hooks.dispatch.call_hooks(
|
||||||
'make_database_dump_pattern',
|
'make_data_source_dump_pattern',
|
||||||
config,
|
config,
|
||||||
repository['path'],
|
repository['path'],
|
||||||
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
|
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
|
||||||
database['name'],
|
data_source['name'],
|
||||||
)[hook_name]
|
)[hook_name]
|
||||||
|
|
||||||
# Kick off a single database extract to stdout.
|
# Kick off a single data source extract to stdout.
|
||||||
extract_process = borgmatic.borg.extract.extract_archive(
|
extract_process = borgmatic.borg.extract.extract_archive(
|
||||||
dry_run=global_arguments.dry_run,
|
dry_run=global_arguments.dry_run,
|
||||||
repository=repository['path'],
|
repository=repository['path'],
|
||||||
|
@ -107,23 +104,23 @@ def restore_single_database(
|
||||||
destination_path='/',
|
destination_path='/',
|
||||||
# A directory format dump isn't a single file, and therefore can't extract
|
# A directory format dump isn't a single file, and therefore can't extract
|
||||||
# to stdout. In this case, the extract_process return value is None.
|
# to stdout. In this case, the extract_process return value is None.
|
||||||
extract_to_stdout=bool(database.get('format') != 'directory'),
|
extract_to_stdout=bool(data_source.get('format') != 'directory'),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Run a single database restore, consuming the extract stdout (if any).
|
# Run a single data source restore, consuming the extract stdout (if any).
|
||||||
borgmatic.hooks.dispatch.call_hooks(
|
borgmatic.hooks.dispatch.call_hooks(
|
||||||
function_name='restore_database_dump',
|
function_name='restore_data_source_dump',
|
||||||
config=config,
|
config=config,
|
||||||
log_prefix=repository['path'],
|
log_prefix=repository['path'],
|
||||||
hook_names=[hook_name],
|
hook_names=[hook_name],
|
||||||
database=database,
|
data_source=data_source,
|
||||||
dry_run=global_arguments.dry_run,
|
dry_run=global_arguments.dry_run,
|
||||||
extract_process=extract_process,
|
extract_process=extract_process,
|
||||||
connection_params=connection_params,
|
connection_params=connection_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def collect_archive_database_names(
|
def collect_archive_data_source_names(
|
||||||
repository,
|
repository,
|
||||||
archive,
|
archive,
|
||||||
config,
|
config,
|
||||||
|
@ -135,60 +132,62 @@ def collect_archive_database_names(
|
||||||
'''
|
'''
|
||||||
Given a local or remote repository path, a resolved archive name, a configuration dict, the
|
Given a local or remote repository path, a resolved archive name, a configuration dict, the
|
||||||
local Borg version, global_arguments an argparse.Namespace, and local and remote Borg paths,
|
local Borg version, global_arguments an argparse.Namespace, and local and remote Borg paths,
|
||||||
query the archive for the names of databases it contains and return them as a dict from hook
|
query the archive for the names of data sources it contains as dumps and return them as a dict
|
||||||
name to a sequence of database names.
|
from hook name to a sequence of data source names.
|
||||||
'''
|
'''
|
||||||
borgmatic_source_directory = os.path.expanduser(
|
borgmatic_source_directory = os.path.expanduser(
|
||||||
config.get(
|
config.get(
|
||||||
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||||
)
|
)
|
||||||
).lstrip('/')
|
).lstrip('/')
|
||||||
parent_dump_path = os.path.expanduser(
|
|
||||||
borgmatic.hooks.dump.make_database_dump_path(borgmatic_source_directory, '*_databases/*/*')
|
|
||||||
)
|
|
||||||
dump_paths = borgmatic.borg.list.capture_archive_listing(
|
dump_paths = borgmatic.borg.list.capture_archive_listing(
|
||||||
repository,
|
repository,
|
||||||
archive,
|
archive,
|
||||||
config,
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
list_path=parent_dump_path,
|
list_paths=[
|
||||||
|
os.path.expanduser(
|
||||||
|
borgmatic.hooks.dump.make_data_source_dump_path(borgmatic_source_directory, pattern)
|
||||||
|
)
|
||||||
|
for pattern in ('*_databases/*/*',)
|
||||||
|
],
|
||||||
local_path=local_path,
|
local_path=local_path,
|
||||||
remote_path=remote_path,
|
remote_path=remote_path,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Determine the database names corresponding to the dumps found in the archive and
|
# Determine the data source names corresponding to the dumps found in the archive and
|
||||||
# add them to restore_names.
|
# add them to restore_names.
|
||||||
archive_database_names = {}
|
archive_data_source_names = {}
|
||||||
|
|
||||||
for dump_path in dump_paths:
|
for dump_path in dump_paths:
|
||||||
try:
|
try:
|
||||||
(hook_name, _, database_name) = dump_path.split(
|
(hook_name, _, data_source_name) = dump_path.split(
|
||||||
borgmatic_source_directory + os.path.sep, 1
|
borgmatic_source_directory + os.path.sep, 1
|
||||||
)[1].split(os.path.sep)[0:3]
|
)[1].split(os.path.sep)[0:3]
|
||||||
except (ValueError, IndexError):
|
except (ValueError, IndexError):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f'{repository}: Ignoring invalid database dump path "{dump_path}" in archive {archive}'
|
f'{repository}: Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
if database_name not in archive_database_names.get(hook_name, []):
|
if data_source_name not in archive_data_source_names.get(hook_name, []):
|
||||||
archive_database_names.setdefault(hook_name, []).extend([database_name])
|
archive_data_source_names.setdefault(hook_name, []).extend([data_source_name])
|
||||||
|
|
||||||
return archive_database_names
|
return archive_data_source_names
|
||||||
|
|
||||||
|
|
||||||
def find_databases_to_restore(requested_database_names, archive_database_names):
|
def find_data_sources_to_restore(requested_data_source_names, archive_data_source_names):
|
||||||
'''
|
'''
|
||||||
Given a sequence of requested database names to restore and a dict of hook name to the names of
|
Given a sequence of requested data source names to restore and a dict of hook name to the names
|
||||||
databases found in an archive, return an expanded sequence of database names to restore,
|
of data sources found in an archive, return an expanded sequence of data source names to
|
||||||
replacing "all" with actual database names as appropriate.
|
restore, replacing "all" with actual data source names as appropriate.
|
||||||
|
|
||||||
Raise ValueError if any of the requested database names cannot be found in the archive.
|
Raise ValueError if any of the requested data source names cannot be found in the archive.
|
||||||
'''
|
'''
|
||||||
# A map from database hook name to the database names to restore for that hook.
|
# A map from data source hook name to the data source names to restore for that hook.
|
||||||
restore_names = (
|
restore_names = (
|
||||||
{UNSPECIFIED_HOOK: requested_database_names}
|
{UNSPECIFIED_HOOK: requested_data_source_names}
|
||||||
if requested_database_names
|
if requested_data_source_names
|
||||||
else {UNSPECIFIED_HOOK: ['all']}
|
else {UNSPECIFIED_HOOK: ['all']}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -197,56 +196,59 @@ def find_databases_to_restore(requested_database_names, archive_database_names):
|
||||||
if 'all' in restore_names[UNSPECIFIED_HOOK]:
|
if 'all' in restore_names[UNSPECIFIED_HOOK]:
|
||||||
restore_names[UNSPECIFIED_HOOK].remove('all')
|
restore_names[UNSPECIFIED_HOOK].remove('all')
|
||||||
|
|
||||||
for hook_name, database_names in archive_database_names.items():
|
for hook_name, data_source_names in archive_data_source_names.items():
|
||||||
restore_names.setdefault(hook_name, []).extend(database_names)
|
restore_names.setdefault(hook_name, []).extend(data_source_names)
|
||||||
|
|
||||||
# If a database is to be restored as part of "all", then remove it from restore names so
|
# If a data source is to be restored as part of "all", then remove it from restore names
|
||||||
# it doesn't get restored twice.
|
# so it doesn't get restored twice.
|
||||||
for database_name in database_names:
|
for data_source_name in data_source_names:
|
||||||
if database_name in restore_names[UNSPECIFIED_HOOK]:
|
if data_source_name in restore_names[UNSPECIFIED_HOOK]:
|
||||||
restore_names[UNSPECIFIED_HOOK].remove(database_name)
|
restore_names[UNSPECIFIED_HOOK].remove(data_source_name)
|
||||||
|
|
||||||
if not restore_names[UNSPECIFIED_HOOK]:
|
if not restore_names[UNSPECIFIED_HOOK]:
|
||||||
restore_names.pop(UNSPECIFIED_HOOK)
|
restore_names.pop(UNSPECIFIED_HOOK)
|
||||||
|
|
||||||
combined_restore_names = set(
|
combined_restore_names = set(
|
||||||
name for database_names in restore_names.values() for name in database_names
|
name for data_source_names in restore_names.values() for name in data_source_names
|
||||||
)
|
)
|
||||||
combined_archive_database_names = set(
|
combined_archive_data_source_names = set(
|
||||||
name for database_names in archive_database_names.values() for name in database_names
|
name
|
||||||
|
for data_source_names in archive_data_source_names.values()
|
||||||
|
for name in data_source_names
|
||||||
)
|
)
|
||||||
|
|
||||||
missing_names = sorted(set(combined_restore_names) - combined_archive_database_names)
|
missing_names = sorted(set(combined_restore_names) - combined_archive_data_source_names)
|
||||||
if missing_names:
|
if missing_names:
|
||||||
joined_names = ', '.join(f'"{name}"' for name in missing_names)
|
joined_names = ', '.join(f'"{name}"' for name in missing_names)
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Cannot restore database{'s' if len(missing_names) > 1 else ''} {joined_names} missing from archive"
|
f"Cannot restore data source{'s' if len(missing_names) > 1 else ''} {joined_names} missing from archive"
|
||||||
)
|
)
|
||||||
|
|
||||||
return restore_names
|
return restore_names
|
||||||
|
|
||||||
|
|
||||||
def ensure_databases_found(restore_names, remaining_restore_names, found_names):
|
def ensure_data_sources_found(restore_names, remaining_restore_names, found_names):
|
||||||
'''
|
'''
|
||||||
Given a dict from hook name to database names to restore, a dict from hook name to remaining
|
Given a dict from hook name to data source names to restore, a dict from hook name to remaining
|
||||||
database names to restore, and a sequence of found (actually restored) database names, raise
|
data source names to restore, and a sequence of found (actually restored) data source names,
|
||||||
ValueError if requested databases to restore were missing from the archive and/or configuration.
|
raise ValueError if requested data source to restore were missing from the archive and/or
|
||||||
|
configuration.
|
||||||
'''
|
'''
|
||||||
combined_restore_names = set(
|
combined_restore_names = set(
|
||||||
name
|
name
|
||||||
for database_names in tuple(restore_names.values())
|
for data_source_names in tuple(restore_names.values())
|
||||||
+ tuple(remaining_restore_names.values())
|
+ tuple(remaining_restore_names.values())
|
||||||
for name in database_names
|
for name in data_source_names
|
||||||
)
|
)
|
||||||
|
|
||||||
if not combined_restore_names and not found_names:
|
if not combined_restore_names and not found_names:
|
||||||
raise ValueError('No databases were found to restore')
|
raise ValueError('No data sources were found to restore')
|
||||||
|
|
||||||
missing_names = sorted(set(combined_restore_names) - set(found_names))
|
missing_names = sorted(set(combined_restore_names) - set(found_names))
|
||||||
if missing_names:
|
if missing_names:
|
||||||
joined_names = ', '.join(f'"{name}"' for name in missing_names)
|
joined_names = ', '.join(f'"{name}"' for name in missing_names)
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Cannot restore database{'s' if len(missing_names) > 1 else ''} {joined_names} missing from borgmatic's configuration"
|
f"Cannot restore data source{'s' if len(missing_names) > 1 else ''} {joined_names} missing from borgmatic's configuration"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -263,7 +265,7 @@ def run_restore(
|
||||||
Run the "restore" action for the given repository, but only if the repository matches the
|
Run the "restore" action for the given repository, but only if the repository matches the
|
||||||
requested repository in restore arguments.
|
requested repository in restore arguments.
|
||||||
|
|
||||||
Raise ValueError if a configured database could not be found to restore.
|
Raise ValueError if a configured data source could not be found to restore.
|
||||||
'''
|
'''
|
||||||
if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
|
if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
|
||||||
repository, restore_arguments.repository
|
repository, restore_arguments.repository
|
||||||
|
@ -271,14 +273,14 @@ def run_restore(
|
||||||
return
|
return
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f'{repository.get("label", repository["path"])}: Restoring databases from archive {restore_arguments.archive}'
|
f'{repository.get("label", repository["path"])}: Restoring data sources from archive {restore_arguments.archive}'
|
||||||
)
|
)
|
||||||
|
|
||||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||||
'remove_database_dumps',
|
'remove_data_source_dumps',
|
||||||
config,
|
config,
|
||||||
repository['path'],
|
repository['path'],
|
||||||
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
|
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -291,7 +293,7 @@ def run_restore(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
archive_database_names = collect_archive_database_names(
|
archive_data_source_names = collect_archive_data_source_names(
|
||||||
repository['path'],
|
repository['path'],
|
||||||
archive_name,
|
archive_name,
|
||||||
config,
|
config,
|
||||||
|
@ -300,7 +302,9 @@ def run_restore(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
|
restore_names = find_data_sources_to_restore(
|
||||||
|
restore_arguments.data_sources, archive_data_source_names
|
||||||
|
)
|
||||||
found_names = set()
|
found_names = set()
|
||||||
remaining_restore_names = {}
|
remaining_restore_names = {}
|
||||||
connection_params = {
|
connection_params = {
|
||||||
|
@ -311,20 +315,20 @@ def run_restore(
|
||||||
'restore_path': restore_arguments.restore_path,
|
'restore_path': restore_arguments.restore_path,
|
||||||
}
|
}
|
||||||
|
|
||||||
for hook_name, database_names in restore_names.items():
|
for hook_name, data_source_names in restore_names.items():
|
||||||
for database_name in database_names:
|
for data_source_name in data_source_names:
|
||||||
found_hook_name, found_database = get_configured_database(
|
found_hook_name, found_data_source = get_configured_data_source(
|
||||||
config, archive_database_names, hook_name, database_name
|
config, archive_data_source_names, hook_name, data_source_name
|
||||||
)
|
)
|
||||||
|
|
||||||
if not found_database:
|
if not found_data_source:
|
||||||
remaining_restore_names.setdefault(found_hook_name or hook_name, []).append(
|
remaining_restore_names.setdefault(found_hook_name or hook_name, []).append(
|
||||||
database_name
|
data_source_name
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
found_names.add(database_name)
|
found_names.add(data_source_name)
|
||||||
restore_single_database(
|
restore_single_data_source(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
|
@ -333,26 +337,26 @@ def run_restore(
|
||||||
remote_path,
|
remote_path,
|
||||||
archive_name,
|
archive_name,
|
||||||
found_hook_name or hook_name,
|
found_hook_name or hook_name,
|
||||||
dict(found_database, **{'schemas': restore_arguments.schemas}),
|
dict(found_data_source, **{'schemas': restore_arguments.schemas}),
|
||||||
connection_params,
|
connection_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
# For any databases that weren't found via exact matches in the configuration, try to fallback
|
# For any data sources that weren't found via exact matches in the configuration, try to
|
||||||
# to "all" entries.
|
# fallback to "all" entries.
|
||||||
for hook_name, database_names in remaining_restore_names.items():
|
for hook_name, data_source_names in remaining_restore_names.items():
|
||||||
for database_name in database_names:
|
for data_source_name in data_source_names:
|
||||||
found_hook_name, found_database = get_configured_database(
|
found_hook_name, found_data_source = get_configured_data_source(
|
||||||
config, archive_database_names, hook_name, database_name, 'all'
|
config, archive_data_source_names, hook_name, data_source_name, 'all'
|
||||||
)
|
)
|
||||||
|
|
||||||
if not found_database:
|
if not found_data_source:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
found_names.add(database_name)
|
found_names.add(data_source_name)
|
||||||
database = copy.copy(found_database)
|
data_source = copy.copy(found_data_source)
|
||||||
database['name'] = database_name
|
data_source['name'] = data_source_name
|
||||||
|
|
||||||
restore_single_database(
|
restore_single_data_source(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
|
@ -361,16 +365,16 @@ def run_restore(
|
||||||
remote_path,
|
remote_path,
|
||||||
archive_name,
|
archive_name,
|
||||||
found_hook_name or hook_name,
|
found_hook_name or hook_name,
|
||||||
dict(database, **{'schemas': restore_arguments.schemas}),
|
dict(data_source, **{'schemas': restore_arguments.schemas}),
|
||||||
connection_params,
|
connection_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||||
'remove_database_dumps',
|
'remove_data_source_dumps',
|
||||||
config,
|
config,
|
||||||
repository['path'],
|
repository['path'],
|
||||||
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
|
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
|
|
||||||
ensure_databases_found(restore_names, remaining_restore_names, found_names)
|
ensure_data_sources_found(restore_names, remaining_restore_names, found_names)
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
import borgmatic.actions.json
|
||||||
import borgmatic.borg.rinfo
|
import borgmatic.borg.rinfo
|
||||||
import borgmatic.config.validate
|
import borgmatic.config.validate
|
||||||
|
|
||||||
|
@ -24,7 +24,7 @@ def run_rinfo(
|
||||||
if rinfo_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
if rinfo_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||||
repository, rinfo_arguments.repository
|
repository, rinfo_arguments.repository
|
||||||
):
|
):
|
||||||
if not rinfo_arguments.json: # pragma: nocover
|
if not rinfo_arguments.json:
|
||||||
logger.answer(
|
logger.answer(
|
||||||
f'{repository.get("label", repository["path"])}: Displaying repository summary information'
|
f'{repository.get("label", repository["path"])}: Displaying repository summary information'
|
||||||
)
|
)
|
||||||
|
@ -38,5 +38,5 @@ def run_rinfo(
|
||||||
local_path=local_path,
|
local_path=local_path,
|
||||||
remote_path=remote_path,
|
remote_path=remote_path,
|
||||||
)
|
)
|
||||||
if json_output: # pragma: nocover
|
if json_output:
|
||||||
yield json.loads(json_output)
|
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
import borgmatic.actions.json
|
||||||
import borgmatic.borg.rlist
|
import borgmatic.borg.rlist
|
||||||
import borgmatic.config.validate
|
import borgmatic.config.validate
|
||||||
|
|
||||||
|
@ -24,7 +24,7 @@ def run_rlist(
|
||||||
if rlist_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
if rlist_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||||
repository, rlist_arguments.repository
|
repository, rlist_arguments.repository
|
||||||
):
|
):
|
||||||
if not rlist_arguments.json: # pragma: nocover
|
if not rlist_arguments.json:
|
||||||
logger.answer(f'{repository.get("label", repository["path"])}: Listing repository')
|
logger.answer(f'{repository.get("label", repository["path"])}: Listing repository')
|
||||||
|
|
||||||
json_output = borgmatic.borg.rlist.list_repository(
|
json_output = borgmatic.borg.rlist.list_repository(
|
||||||
|
@ -36,5 +36,5 @@ def run_rlist(
|
||||||
local_path=local_path,
|
local_path=local_path,
|
||||||
remote_path=remote_path,
|
remote_path=remote_path,
|
||||||
)
|
)
|
||||||
if json_output: # pragma: nocover
|
if json_output:
|
||||||
yield json.loads(json_output)
|
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import logging
|
import logging
|
||||||
|
import shlex
|
||||||
|
|
||||||
import borgmatic.commands.arguments
|
import borgmatic.commands.arguments
|
||||||
import borgmatic.logger
|
import borgmatic.logger
|
||||||
|
@ -56,9 +57,8 @@ def run_arbitrary_borg(
|
||||||
)
|
)
|
||||||
|
|
||||||
return execute_command(
|
return execute_command(
|
||||||
full_command,
|
tuple(shlex.quote(part) for part in full_command),
|
||||||
output_file=DO_NOT_CAPTURE,
|
output_file=DO_NOT_CAPTURE,
|
||||||
borg_local_path=local_path,
|
|
||||||
shell=True,
|
shell=True,
|
||||||
extra_environment=dict(
|
extra_environment=dict(
|
||||||
(environment.make_environment(config) or {}),
|
(environment.make_environment(config) or {}),
|
||||||
|
@ -67,4 +67,6 @@ def run_arbitrary_borg(
|
||||||
'ARCHIVE': archive if archive else '',
|
'ARCHIVE': archive if archive else '',
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
|
|
|
@ -34,4 +34,9 @@ def break_lock(
|
||||||
)
|
)
|
||||||
|
|
||||||
borg_environment = environment.make_environment(config)
|
borg_environment = environment.make_environment(config)
|
||||||
execute_command(full_command, borg_local_path=local_path, extra_environment=borg_environment)
|
execute_command(
|
||||||
|
full_command,
|
||||||
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
|
)
|
||||||
|
|
|
@ -1,163 +1,26 @@
|
||||||
import argparse
|
import argparse
|
||||||
import datetime
|
|
||||||
import hashlib
|
|
||||||
import itertools
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
|
|
||||||
from borgmatic.borg import environment, extract, feature, flags, rinfo, state
|
from borgmatic.borg import environment, feature, flags, rinfo
|
||||||
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
|
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
|
||||||
|
|
||||||
DEFAULT_CHECKS = (
|
|
||||||
{'name': 'repository', 'frequency': '1 month'},
|
|
||||||
{'name': 'archives', 'frequency': '1 month'},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def parse_checks(config, only_checks=None):
|
def make_archive_filter_flags(local_borg_version, config, checks, check_arguments):
|
||||||
'''
|
'''
|
||||||
Given a configuration dict with a "checks" sequence of dicts and an optional list of override
|
Given the local Borg version, a configuration dict, a parsed sequence of checks, and check
|
||||||
checks, return a tuple of named checks to run.
|
arguments as an argparse.Namespace instance, transform the checks into tuple of command-line
|
||||||
|
flags for filtering archives in a check command.
|
||||||
|
|
||||||
For example, given a config of:
|
If "check_last" is set in the configuration and "archives" is in checks, then include a "--last"
|
||||||
|
flag. And if "prefix" is set in configuration and "archives" is in checks, then include a
|
||||||
{'checks': ({'name': 'repository'}, {'name': 'archives'})}
|
"--match-archives" flag.
|
||||||
|
|
||||||
This will be returned as:
|
|
||||||
|
|
||||||
('repository', 'archives')
|
|
||||||
|
|
||||||
If no "checks" option is present in the config, return the DEFAULT_CHECKS. If a checks value
|
|
||||||
has a name of "disabled", return an empty tuple, meaning that no checks should be run.
|
|
||||||
'''
|
'''
|
||||||
checks = only_checks or tuple(
|
check_last = config.get('check_last', None)
|
||||||
check_config['name'] for check_config in (config.get('checks', None) or DEFAULT_CHECKS)
|
prefix = config.get('prefix')
|
||||||
)
|
|
||||||
checks = tuple(check.lower() for check in checks)
|
|
||||||
if 'disabled' in checks:
|
|
||||||
if len(checks) > 1:
|
|
||||||
logger.warning(
|
|
||||||
'Multiple checks are configured, but one of them is "disabled"; not running any checks'
|
|
||||||
)
|
|
||||||
return ()
|
|
||||||
|
|
||||||
return checks
|
|
||||||
|
|
||||||
|
|
||||||
def parse_frequency(frequency):
|
|
||||||
'''
|
|
||||||
Given a frequency string with a number and a unit of time, return a corresponding
|
|
||||||
datetime.timedelta instance or None if the frequency is None or "always".
|
|
||||||
|
|
||||||
For instance, given "3 weeks", return datetime.timedelta(weeks=3)
|
|
||||||
|
|
||||||
Raise ValueError if the given frequency cannot be parsed.
|
|
||||||
'''
|
|
||||||
if not frequency:
|
|
||||||
return None
|
|
||||||
|
|
||||||
frequency = frequency.strip().lower()
|
|
||||||
|
|
||||||
if frequency == 'always':
|
|
||||||
return None
|
|
||||||
|
|
||||||
try:
|
|
||||||
number, time_unit = frequency.split(' ')
|
|
||||||
number = int(number)
|
|
||||||
except ValueError:
|
|
||||||
raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
|
|
||||||
|
|
||||||
if not time_unit.endswith('s'):
|
|
||||||
time_unit += 's'
|
|
||||||
|
|
||||||
if time_unit == 'months':
|
|
||||||
number *= 30
|
|
||||||
time_unit = 'days'
|
|
||||||
elif time_unit == 'years':
|
|
||||||
number *= 365
|
|
||||||
time_unit = 'days'
|
|
||||||
|
|
||||||
try:
|
|
||||||
return datetime.timedelta(**{time_unit: number})
|
|
||||||
except TypeError:
|
|
||||||
raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
|
|
||||||
|
|
||||||
|
|
||||||
def filter_checks_on_frequency(
|
|
||||||
config,
|
|
||||||
borg_repository_id,
|
|
||||||
checks,
|
|
||||||
force,
|
|
||||||
archives_check_id=None,
|
|
||||||
):
|
|
||||||
'''
|
|
||||||
Given a configuration dict with a "checks" sequence of dicts, a Borg repository ID, a sequence
|
|
||||||
of checks, whether to force checks to run, and an ID for the archives check potentially being
|
|
||||||
run (if any), filter down those checks based on the configured "frequency" for each check as
|
|
||||||
compared to its check time file.
|
|
||||||
|
|
||||||
In other words, a check whose check time file's timestamp is too new (based on the configured
|
|
||||||
frequency) will get cut from the returned sequence of checks. Example:
|
|
||||||
|
|
||||||
config = {
|
|
||||||
'checks': [
|
|
||||||
{
|
|
||||||
'name': 'archives',
|
|
||||||
'frequency': '2 weeks',
|
|
||||||
},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
When this function is called with that config and "archives" in checks, "archives" will get
|
|
||||||
filtered out of the returned result if its check time file is newer than 2 weeks old, indicating
|
|
||||||
that it's not yet time to run that check again.
|
|
||||||
|
|
||||||
Raise ValueError if a frequency cannot be parsed.
|
|
||||||
'''
|
|
||||||
filtered_checks = list(checks)
|
|
||||||
|
|
||||||
if force:
|
|
||||||
return tuple(filtered_checks)
|
|
||||||
|
|
||||||
for check_config in config.get('checks', DEFAULT_CHECKS):
|
|
||||||
check = check_config['name']
|
|
||||||
if checks and check not in checks:
|
|
||||||
continue
|
|
||||||
|
|
||||||
frequency_delta = parse_frequency(check_config.get('frequency'))
|
|
||||||
if not frequency_delta:
|
|
||||||
continue
|
|
||||||
|
|
||||||
check_time = probe_for_check_time(config, borg_repository_id, check, archives_check_id)
|
|
||||||
if not check_time:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# If we've not yet reached the time when the frequency dictates we're ready for another
|
|
||||||
# check, skip this check.
|
|
||||||
if datetime.datetime.now() < check_time + frequency_delta:
|
|
||||||
remaining = check_time + frequency_delta - datetime.datetime.now()
|
|
||||||
logger.info(
|
|
||||||
f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
|
|
||||||
)
|
|
||||||
filtered_checks.remove(check)
|
|
||||||
|
|
||||||
return tuple(filtered_checks)
|
|
||||||
|
|
||||||
|
|
||||||
def make_archive_filter_flags(local_borg_version, config, checks, check_last=None, prefix=None):
|
|
||||||
'''
|
|
||||||
Given the local Borg version, a configuration dict, a parsed sequence of checks, the check last
|
|
||||||
value, and a consistency check prefix, transform the checks into tuple of command-line flags for
|
|
||||||
filtering archives in a check command.
|
|
||||||
|
|
||||||
If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if
|
|
||||||
a prefix value is given and "archives" is in checks, then include a "--match-archives" flag.
|
|
||||||
'''
|
|
||||||
if 'archives' in checks or 'data' in checks:
|
if 'archives' in checks or 'data' in checks:
|
||||||
return (('--last', str(check_last)) if check_last else ()) + (
|
return (('--last', str(check_last)) if check_last else ()) + (
|
||||||
(
|
(
|
||||||
|
@ -168,7 +31,7 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_last=Non
|
||||||
if prefix
|
if prefix
|
||||||
else (
|
else (
|
||||||
flags.make_match_archives_flags(
|
flags.make_match_archives_flags(
|
||||||
config.get('match_archives'),
|
check_arguments.match_archives or config.get('match_archives'),
|
||||||
config.get('archive_name_format'),
|
config.get('archive_name_format'),
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
)
|
)
|
||||||
|
@ -187,21 +50,10 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_last=Non
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
|
|
||||||
def make_archives_check_id(archive_filter_flags):
|
|
||||||
'''
|
|
||||||
Given a sequence of flags to filter archives, return a unique hash corresponding to those
|
|
||||||
particular flags. If there are no flags, return None.
|
|
||||||
'''
|
|
||||||
if not archive_filter_flags:
|
|
||||||
return None
|
|
||||||
|
|
||||||
return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def make_check_flags(checks, archive_filter_flags):
|
def make_check_flags(checks, archive_filter_flags):
|
||||||
'''
|
'''
|
||||||
Given a parsed sequence of checks and a sequence of flags to filter archives, transform the
|
Given a parsed checks set and a sequence of flags to filter archives,
|
||||||
checks into tuple of command-line check flags.
|
transform the checks into tuple of command-line check flags.
|
||||||
|
|
||||||
For example, given parsed checks of:
|
For example, given parsed checks of:
|
||||||
|
|
||||||
|
@ -216,13 +68,13 @@ def make_check_flags(checks, archive_filter_flags):
|
||||||
'''
|
'''
|
||||||
if 'data' in checks:
|
if 'data' in checks:
|
||||||
data_flags = ('--verify-data',)
|
data_flags = ('--verify-data',)
|
||||||
checks += ('archives',)
|
checks.update({'archives'})
|
||||||
else:
|
else:
|
||||||
data_flags = ()
|
data_flags = ()
|
||||||
|
|
||||||
common_flags = (archive_filter_flags if 'archives' in checks else ()) + data_flags
|
common_flags = (archive_filter_flags if 'archives' in checks else ()) + data_flags
|
||||||
|
|
||||||
if {'repository', 'archives'}.issubset(set(checks)):
|
if {'repository', 'archives'}.issubset(checks):
|
||||||
return common_flags
|
return common_flags
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
@ -231,147 +83,17 @@ def make_check_flags(checks, archive_filter_flags):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def make_check_time_path(config, borg_repository_id, check_type, archives_check_id=None):
|
def get_repository_id(
|
||||||
'''
|
repository_path, config, local_borg_version, global_arguments, local_path, remote_path
|
||||||
Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
|
|
||||||
"archives", etc.), and a unique hash of the archives filter flags, return a path for recording
|
|
||||||
that check's time (the time of that check last occurring).
|
|
||||||
'''
|
|
||||||
borgmatic_source_directory = os.path.expanduser(
|
|
||||||
config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY)
|
|
||||||
)
|
|
||||||
|
|
||||||
if check_type in ('archives', 'data'):
|
|
||||||
return os.path.join(
|
|
||||||
borgmatic_source_directory,
|
|
||||||
'checks',
|
|
||||||
borg_repository_id,
|
|
||||||
check_type,
|
|
||||||
archives_check_id if archives_check_id else 'all',
|
|
||||||
)
|
|
||||||
|
|
||||||
return os.path.join(
|
|
||||||
borgmatic_source_directory,
|
|
||||||
'checks',
|
|
||||||
borg_repository_id,
|
|
||||||
check_type,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_check_time(path): # pragma: no cover
|
|
||||||
'''
|
|
||||||
Record a check time of now as the modification time of the given path.
|
|
||||||
'''
|
|
||||||
logger.debug(f'Writing check time at {path}')
|
|
||||||
|
|
||||||
os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
|
|
||||||
pathlib.Path(path, mode=0o600).touch()
|
|
||||||
|
|
||||||
|
|
||||||
def read_check_time(path):
|
|
||||||
'''
|
|
||||||
Return the check time based on the modification time of the given path. Return None if the path
|
|
||||||
doesn't exist.
|
|
||||||
'''
|
|
||||||
logger.debug(f'Reading check time from {path}')
|
|
||||||
|
|
||||||
try:
|
|
||||||
return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
|
|
||||||
except FileNotFoundError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
|
|
||||||
'''
|
|
||||||
Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
|
|
||||||
"archives", etc.), and a unique hash of the archives filter flags, return a the corresponding
|
|
||||||
check time or None if such a check time does not exist.
|
|
||||||
|
|
||||||
When the check type is "archives" or "data", this function probes two different paths to find
|
|
||||||
the check time, e.g.:
|
|
||||||
|
|
||||||
~/.borgmatic/checks/1234567890/archives/9876543210
|
|
||||||
~/.borgmatic/checks/1234567890/archives/all
|
|
||||||
|
|
||||||
... and returns the maximum modification time of the files found (if any). The first path
|
|
||||||
represents a more specific archives check time (a check on a subset of archives), and the second
|
|
||||||
is a fallback to the last "all" archives check.
|
|
||||||
|
|
||||||
For other check types, this function reads from a single check time path, e.g.:
|
|
||||||
|
|
||||||
~/.borgmatic/checks/1234567890/repository
|
|
||||||
'''
|
|
||||||
check_times = (
|
|
||||||
read_check_time(group[0])
|
|
||||||
for group in itertools.groupby(
|
|
||||||
(
|
|
||||||
make_check_time_path(config, borg_repository_id, check, archives_check_id),
|
|
||||||
make_check_time_path(config, borg_repository_id, check),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
return max(check_time for check_time in check_times if check_time)
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade_check_times(config, borg_repository_id):
|
|
||||||
'''
|
|
||||||
Given a configuration dict and a Borg repository ID, upgrade any corresponding check times on
|
|
||||||
disk from old-style paths to new-style paths.
|
|
||||||
|
|
||||||
Currently, the only upgrade performed is renaming an archive or data check path that looks like:
|
|
||||||
|
|
||||||
~/.borgmatic/checks/1234567890/archives
|
|
||||||
|
|
||||||
to:
|
|
||||||
|
|
||||||
~/.borgmatic/checks/1234567890/archives/all
|
|
||||||
'''
|
|
||||||
for check_type in ('archives', 'data'):
|
|
||||||
new_path = make_check_time_path(config, borg_repository_id, check_type, 'all')
|
|
||||||
old_path = os.path.dirname(new_path)
|
|
||||||
temporary_path = f'{old_path}.temp'
|
|
||||||
|
|
||||||
if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
|
|
||||||
continue
|
|
||||||
|
|
||||||
logger.debug(f'Upgrading archives check time from {old_path} to {new_path}')
|
|
||||||
|
|
||||||
try:
|
|
||||||
os.rename(old_path, temporary_path)
|
|
||||||
except FileNotFoundError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
os.mkdir(old_path)
|
|
||||||
os.rename(temporary_path, new_path)
|
|
||||||
|
|
||||||
|
|
||||||
def check_archives(
|
|
||||||
repository_path,
|
|
||||||
config,
|
|
||||||
local_borg_version,
|
|
||||||
global_arguments,
|
|
||||||
local_path='borg',
|
|
||||||
remote_path=None,
|
|
||||||
progress=None,
|
|
||||||
repair=None,
|
|
||||||
only_checks=None,
|
|
||||||
force=None,
|
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given a local or remote repository path, a configuration dict, local/remote commands to run,
|
Given a local or remote repository path, a configuration dict, the local Borg version, global
|
||||||
whether to include progress information, whether to attempt a repair, and an optional list of
|
arguments, and local/remote commands to run, return the corresponding Borg repository ID.
|
||||||
checks to use instead of configured checks, check the contained Borg archives for consistency.
|
|
||||||
|
|
||||||
If there are no consistency checks to run, skip running them.
|
Raise ValueError if the Borg repository ID cannot be determined.
|
||||||
|
|
||||||
Raises ValueError if the Borg repository ID cannot be determined.
|
|
||||||
'''
|
'''
|
||||||
try:
|
try:
|
||||||
borg_repository_id = json.loads(
|
return json.loads(
|
||||||
rinfo.display_repository_info(
|
rinfo.display_repository_info(
|
||||||
repository_path,
|
repository_path,
|
||||||
config,
|
config,
|
||||||
|
@ -385,72 +107,63 @@ def check_archives(
|
||||||
except (json.JSONDecodeError, KeyError):
|
except (json.JSONDecodeError, KeyError):
|
||||||
raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
|
raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
|
||||||
|
|
||||||
upgrade_check_times(config, borg_repository_id)
|
|
||||||
|
|
||||||
check_last = config.get('check_last', None)
|
def check_archives(
|
||||||
prefix = config.get('prefix')
|
repository_path,
|
||||||
configured_checks = parse_checks(config, only_checks)
|
config,
|
||||||
lock_wait = None
|
local_borg_version,
|
||||||
|
check_arguments,
|
||||||
|
global_arguments,
|
||||||
|
checks,
|
||||||
|
archive_filter_flags,
|
||||||
|
local_path='borg',
|
||||||
|
remote_path=None,
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Given a local or remote repository path, a configuration dict, the local Borg version, check
|
||||||
|
arguments as an argparse.Namespace instance, global arguments, a set of named Borg checks to run
|
||||||
|
(some combination "repository", "archives", and/or "data"), archive filter flags, and
|
||||||
|
local/remote commands to run, check the contained Borg archives for consistency.
|
||||||
|
'''
|
||||||
|
lock_wait = config.get('lock_wait')
|
||||||
extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
|
extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
|
||||||
archive_filter_flags = make_archive_filter_flags(
|
|
||||||
local_borg_version, config, configured_checks, check_last, prefix
|
|
||||||
)
|
|
||||||
archives_check_id = make_archives_check_id(archive_filter_flags)
|
|
||||||
|
|
||||||
checks = filter_checks_on_frequency(
|
verbosity_flags = ()
|
||||||
config,
|
if logger.isEnabledFor(logging.INFO):
|
||||||
borg_repository_id,
|
verbosity_flags = ('--info',)
|
||||||
configured_checks,
|
if logger.isEnabledFor(logging.DEBUG):
|
||||||
force,
|
verbosity_flags = ('--debug', '--show-rc')
|
||||||
archives_check_id,
|
|
||||||
|
full_command = (
|
||||||
|
(local_path, 'check')
|
||||||
|
+ (('--repair',) if check_arguments.repair else ())
|
||||||
|
+ make_check_flags(checks, archive_filter_flags)
|
||||||
|
+ (('--remote-path', remote_path) if remote_path else ())
|
||||||
|
+ (('--log-json',) if global_arguments.log_json else ())
|
||||||
|
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||||
|
+ verbosity_flags
|
||||||
|
+ (('--progress',) if check_arguments.progress else ())
|
||||||
|
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
|
||||||
|
+ flags.make_repository_flags(repository_path, local_borg_version)
|
||||||
)
|
)
|
||||||
|
|
||||||
if set(checks).intersection({'repository', 'archives', 'data'}):
|
borg_environment = environment.make_environment(config)
|
||||||
lock_wait = config.get('lock_wait')
|
borg_exit_codes = config.get('borg_exit_codes')
|
||||||
|
|
||||||
verbosity_flags = ()
|
# The Borg repair option triggers an interactive prompt, which won't work when output is
|
||||||
if logger.isEnabledFor(logging.INFO):
|
# captured. And progress messes with the terminal directly.
|
||||||
verbosity_flags = ('--info',)
|
if check_arguments.repair or check_arguments.progress:
|
||||||
if logger.isEnabledFor(logging.DEBUG):
|
execute_command(
|
||||||
verbosity_flags = ('--debug', '--show-rc')
|
full_command,
|
||||||
|
output_file=DO_NOT_CAPTURE,
|
||||||
full_command = (
|
extra_environment=borg_environment,
|
||||||
(local_path, 'check')
|
borg_local_path=local_path,
|
||||||
+ (('--repair',) if repair else ())
|
borg_exit_codes=borg_exit_codes,
|
||||||
+ make_check_flags(checks, archive_filter_flags)
|
|
||||||
+ (('--remote-path', remote_path) if remote_path else ())
|
|
||||||
+ (('--log-json',) if global_arguments.log_json else ())
|
|
||||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
|
||||||
+ verbosity_flags
|
|
||||||
+ (('--progress',) if progress else ())
|
|
||||||
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
|
|
||||||
+ flags.make_repository_flags(repository_path, local_borg_version)
|
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
borg_environment = environment.make_environment(config)
|
execute_command(
|
||||||
|
full_command,
|
||||||
# The Borg repair option triggers an interactive prompt, which won't work when output is
|
extra_environment=borg_environment,
|
||||||
# captured. And progress messes with the terminal directly.
|
borg_local_path=local_path,
|
||||||
if repair or progress:
|
borg_exit_codes=borg_exit_codes,
|
||||||
execute_command(
|
|
||||||
full_command, output_file=DO_NOT_CAPTURE, extra_environment=borg_environment
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
execute_command(full_command, extra_environment=borg_environment)
|
|
||||||
|
|
||||||
for check in checks:
|
|
||||||
write_check_time(
|
|
||||||
make_check_time_path(config, borg_repository_id, check, archives_check_id)
|
|
||||||
)
|
|
||||||
|
|
||||||
if 'extract' in checks:
|
|
||||||
extract.extract_last_archive_dry_run(
|
|
||||||
config,
|
|
||||||
local_borg_version,
|
|
||||||
global_arguments,
|
|
||||||
repository_path,
|
|
||||||
lock_wait,
|
|
||||||
local_path,
|
|
||||||
remote_path,
|
|
||||||
)
|
)
|
||||||
write_check_time(make_check_time_path(config, borg_repository_id, 'extract'))
|
|
||||||
|
|
|
@ -48,6 +48,7 @@ def compact_segments(
|
||||||
execute_command(
|
execute_command(
|
||||||
full_command,
|
full_command,
|
||||||
output_log_level=logging.INFO,
|
output_log_level=logging.INFO,
|
||||||
borg_local_path=local_path,
|
|
||||||
extra_environment=environment.make_environment(config),
|
extra_environment=environment.make_environment(config),
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
|
|
|
@ -215,9 +215,6 @@ def make_list_filter_flags(local_borg_version, dry_run):
|
||||||
return f'{base_flags}-'
|
return f'{base_flags}-'
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003
|
|
||||||
|
|
||||||
|
|
||||||
def collect_borgmatic_source_directories(borgmatic_source_directory):
|
def collect_borgmatic_source_directories(borgmatic_source_directory):
|
||||||
'''
|
'''
|
||||||
Return a list of borgmatic-specific source directories used for state like database backups.
|
Return a list of borgmatic-specific source directories used for state like database backups.
|
||||||
|
@ -275,14 +272,14 @@ def any_parent_directories(path, candidate_parents):
|
||||||
|
|
||||||
|
|
||||||
def collect_special_file_paths(
|
def collect_special_file_paths(
|
||||||
create_command, local_path, working_directory, borg_environment, skip_directories
|
create_command, config, local_path, working_directory, borg_environment, skip_directories
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given a Borg create command as a tuple, a local Borg path, a working directory, a dict of
|
Given a Borg create command as a tuple, a configuration dict, a local Borg path, a working
|
||||||
environment variables to pass to Borg, and a sequence of parent directories to skip, collect the
|
directory, a dict of environment variables to pass to Borg, and a sequence of parent directories
|
||||||
paths for any special files (character devices, block devices, and named pipes / FIFOs) that
|
to skip, collect the paths for any special files (character devices, block devices, and named
|
||||||
Borg would encounter during a create. These are all paths that could cause Borg to hang if its
|
pipes / FIFOs) that Borg would encounter during a create. These are all paths that could cause
|
||||||
--read-special flag is used.
|
Borg to hang if its --read-special flag is used.
|
||||||
'''
|
'''
|
||||||
# Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
|
# Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
|
||||||
# files including any named pipe we've created.
|
# files including any named pipe we've created.
|
||||||
|
@ -293,6 +290,7 @@ def collect_special_file_paths(
|
||||||
working_directory=working_directory,
|
working_directory=working_directory,
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
borg_local_path=local_path,
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
|
|
||||||
paths = tuple(
|
paths = tuple(
|
||||||
|
@ -322,43 +320,37 @@ def check_all_source_directories_exist(source_directories):
|
||||||
raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
|
raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
|
||||||
|
|
||||||
|
|
||||||
def create_archive(
|
def make_base_create_command(
|
||||||
dry_run,
|
dry_run,
|
||||||
repository_path,
|
repository_path,
|
||||||
config,
|
config,
|
||||||
|
config_paths,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
|
borgmatic_source_directories,
|
||||||
local_path='borg',
|
local_path='borg',
|
||||||
remote_path=None,
|
remote_path=None,
|
||||||
progress=False,
|
progress=False,
|
||||||
stats=False,
|
|
||||||
json=False,
|
json=False,
|
||||||
list_files=False,
|
list_files=False,
|
||||||
stream_processes=None,
|
stream_processes=None,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given vebosity/dry-run flags, a local or remote repository path, and a configuration dict,
|
Given vebosity/dry-run flags, a local or remote repository path, a configuration dict, a
|
||||||
create a Borg archive and return Borg's JSON output (if any).
|
sequence of loaded configuration paths, the local Borg version, global arguments as an
|
||||||
|
argparse.Namespace instance, and a sequence of borgmatic source directories, return a tuple of
|
||||||
If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
|
(base Borg create command flags, Borg create command positional arguments, open pattern file
|
||||||
create command while also triggering the given processes to produce output.
|
handle, open exclude file handle).
|
||||||
'''
|
'''
|
||||||
borgmatic.logger.add_custom_log_levels()
|
|
||||||
borgmatic_source_directories = expand_directories(
|
|
||||||
collect_borgmatic_source_directories(config.get('borgmatic_source_directory'))
|
|
||||||
)
|
|
||||||
if config.get('source_directories_must_exist', False):
|
if config.get('source_directories_must_exist', False):
|
||||||
check_all_source_directories_exist(config.get('source_directories'))
|
check_all_source_directories_exist(config.get('source_directories'))
|
||||||
|
|
||||||
sources = deduplicate_directories(
|
sources = deduplicate_directories(
|
||||||
map_directories_to_devices(
|
map_directories_to_devices(
|
||||||
expand_directories(
|
expand_directories(
|
||||||
tuple(config.get('source_directories', ()))
|
tuple(config.get('source_directories', ()))
|
||||||
+ borgmatic_source_directories
|
+ borgmatic_source_directories
|
||||||
+ tuple(
|
+ tuple(config_paths if config.get('store_config_files', True) else ())
|
||||||
global_arguments.used_config_paths
|
|
||||||
if config.get('store_config_files', True)
|
|
||||||
else ()
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
additional_directory_devices=map_directories_to_devices(
|
additional_directory_devices=map_directories_to_devices(
|
||||||
|
@ -368,11 +360,6 @@ def create_archive(
|
||||||
|
|
||||||
ensure_files_readable(config.get('patterns_from'), config.get('exclude_from'))
|
ensure_files_readable(config.get('patterns_from'), config.get('exclude_from'))
|
||||||
|
|
||||||
try:
|
|
||||||
working_directory = os.path.expanduser(config.get('working_directory'))
|
|
||||||
except TypeError:
|
|
||||||
working_directory = None
|
|
||||||
|
|
||||||
pattern_file = (
|
pattern_file = (
|
||||||
write_pattern_file(config.get('patterns'), sources)
|
write_pattern_file(config.get('patterns'), sources)
|
||||||
if config.get('patterns') or config.get('patterns_from')
|
if config.get('patterns') or config.get('patterns_from')
|
||||||
|
@ -388,7 +375,7 @@ def create_archive(
|
||||||
lock_wait = config.get('lock_wait', None)
|
lock_wait = config.get('lock_wait', None)
|
||||||
list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
|
list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
|
||||||
files_cache = config.get('files_cache')
|
files_cache = config.get('files_cache')
|
||||||
archive_name_format = config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
|
archive_name_format = config.get('archive_name_format', flags.DEFAULT_ARCHIVE_NAME_FORMAT)
|
||||||
extra_borg_options = config.get('extra_borg_options', {}).get('create', '')
|
extra_borg_options = config.get('extra_borg_options', {}).get('create', '')
|
||||||
|
|
||||||
if feature.available(feature.Feature.ATIME, local_borg_version):
|
if feature.available(feature.Feature.ATIME, local_borg_version):
|
||||||
|
@ -415,11 +402,6 @@ def create_archive(
|
||||||
('--remote-ratelimit', str(upload_rate_limit)) if upload_rate_limit else ()
|
('--remote-ratelimit', str(upload_rate_limit)) if upload_rate_limit else ()
|
||||||
)
|
)
|
||||||
|
|
||||||
if stream_processes and config.get('read_special') is False:
|
|
||||||
logger.warning(
|
|
||||||
f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
|
|
||||||
)
|
|
||||||
|
|
||||||
create_flags = (
|
create_flags = (
|
||||||
tuple(local_path.split(' '))
|
tuple(local_path.split(' '))
|
||||||
+ ('create',)
|
+ ('create',)
|
||||||
|
@ -455,25 +437,23 @@ def create_archive(
|
||||||
repository_path, archive_name_format, local_borg_version
|
repository_path, archive_name_format, local_borg_version
|
||||||
) + (sources if not pattern_file else ())
|
) + (sources if not pattern_file else ())
|
||||||
|
|
||||||
if json:
|
|
||||||
output_log_level = None
|
|
||||||
elif list_files or (stats and not dry_run):
|
|
||||||
output_log_level = logging.ANSWER
|
|
||||||
else:
|
|
||||||
output_log_level = logging.INFO
|
|
||||||
|
|
||||||
# The progress output isn't compatible with captured and logged output, as progress messes with
|
|
||||||
# the terminal directly.
|
|
||||||
output_file = DO_NOT_CAPTURE if progress else None
|
|
||||||
|
|
||||||
borg_environment = environment.make_environment(config)
|
|
||||||
|
|
||||||
# If database hooks are enabled (as indicated by streaming processes), exclude files that might
|
# If database hooks are enabled (as indicated by streaming processes), exclude files that might
|
||||||
# cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
|
# cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
|
||||||
if stream_processes and not config.get('read_special'):
|
if stream_processes and not config.get('read_special'):
|
||||||
|
logger.warning(
|
||||||
|
f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
working_directory = os.path.expanduser(config.get('working_directory'))
|
||||||
|
except TypeError:
|
||||||
|
working_directory = None
|
||||||
|
|
||||||
|
borg_environment = environment.make_environment(config)
|
||||||
|
|
||||||
logger.debug(f'{repository_path}: Collecting special file paths')
|
logger.debug(f'{repository_path}: Collecting special file paths')
|
||||||
special_file_paths = collect_special_file_paths(
|
special_file_paths = collect_special_file_paths(
|
||||||
create_flags + create_positional_arguments,
|
create_flags + create_positional_arguments,
|
||||||
|
config,
|
||||||
local_path,
|
local_path,
|
||||||
working_directory,
|
working_directory,
|
||||||
borg_environment,
|
borg_environment,
|
||||||
|
@ -492,6 +472,73 @@ def create_archive(
|
||||||
)
|
)
|
||||||
create_flags += make_exclude_flags(config, exclude_file.name)
|
create_flags += make_exclude_flags(config, exclude_file.name)
|
||||||
|
|
||||||
|
return (create_flags, create_positional_arguments, pattern_file, exclude_file)
|
||||||
|
|
||||||
|
|
||||||
|
def create_archive(
|
||||||
|
dry_run,
|
||||||
|
repository_path,
|
||||||
|
config,
|
||||||
|
config_paths,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
local_path='borg',
|
||||||
|
remote_path=None,
|
||||||
|
progress=False,
|
||||||
|
stats=False,
|
||||||
|
json=False,
|
||||||
|
list_files=False,
|
||||||
|
stream_processes=None,
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Given vebosity/dry-run flags, a local or remote repository path, a configuration dict, a
|
||||||
|
sequence of loaded configuration paths, the local Borg version, and global arguments as an
|
||||||
|
argparse.Namespace instance, create a Borg archive and return Borg's JSON output (if any).
|
||||||
|
|
||||||
|
If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
|
||||||
|
create command while also triggering the given processes to produce output.
|
||||||
|
'''
|
||||||
|
borgmatic.logger.add_custom_log_levels()
|
||||||
|
borgmatic_source_directories = expand_directories(
|
||||||
|
collect_borgmatic_source_directories(config.get('borgmatic_source_directory'))
|
||||||
|
)
|
||||||
|
|
||||||
|
(create_flags, create_positional_arguments, pattern_file, exclude_file) = (
|
||||||
|
make_base_create_command(
|
||||||
|
dry_run,
|
||||||
|
repository_path,
|
||||||
|
config,
|
||||||
|
config_paths,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
borgmatic_source_directories,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
progress,
|
||||||
|
json,
|
||||||
|
list_files,
|
||||||
|
stream_processes,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if json:
|
||||||
|
output_log_level = None
|
||||||
|
elif list_files or (stats and not dry_run):
|
||||||
|
output_log_level = logging.ANSWER
|
||||||
|
else:
|
||||||
|
output_log_level = logging.INFO
|
||||||
|
|
||||||
|
# The progress output isn't compatible with captured and logged output, as progress messes with
|
||||||
|
# the terminal directly.
|
||||||
|
output_file = DO_NOT_CAPTURE if progress else None
|
||||||
|
|
||||||
|
try:
|
||||||
|
working_directory = os.path.expanduser(config.get('working_directory'))
|
||||||
|
except TypeError:
|
||||||
|
working_directory = None
|
||||||
|
|
||||||
|
borg_environment = environment.make_environment(config)
|
||||||
|
|
||||||
create_flags += (
|
create_flags += (
|
||||||
(('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
|
(('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
|
||||||
+ (('--stats',) if stats and not json and not dry_run else ())
|
+ (('--stats',) if stats and not json and not dry_run else ())
|
||||||
|
@ -499,6 +546,7 @@ def create_archive(
|
||||||
+ (('--progress',) if progress else ())
|
+ (('--progress',) if progress else ())
|
||||||
+ (('--json',) if json else ())
|
+ (('--json',) if json else ())
|
||||||
)
|
)
|
||||||
|
borg_exit_codes = config.get('borg_exit_codes')
|
||||||
|
|
||||||
if stream_processes:
|
if stream_processes:
|
||||||
return execute_command_with_processes(
|
return execute_command_with_processes(
|
||||||
|
@ -506,9 +554,10 @@ def create_archive(
|
||||||
stream_processes,
|
stream_processes,
|
||||||
output_log_level,
|
output_log_level,
|
||||||
output_file,
|
output_file,
|
||||||
borg_local_path=local_path,
|
|
||||||
working_directory=working_directory,
|
working_directory=working_directory,
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
elif output_log_level is None:
|
elif output_log_level is None:
|
||||||
return execute_command_and_capture_output(
|
return execute_command_and_capture_output(
|
||||||
|
@ -516,13 +565,15 @@ def create_archive(
|
||||||
working_directory=working_directory,
|
working_directory=working_directory,
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
borg_local_path=local_path,
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
execute_command(
|
execute_command(
|
||||||
create_flags + create_positional_arguments,
|
create_flags + create_positional_arguments,
|
||||||
output_log_level,
|
output_log_level,
|
||||||
output_file,
|
output_file,
|
||||||
borg_local_path=local_path,
|
|
||||||
working_directory=working_directory,
|
working_directory=working_directory,
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
|
|
|
@ -50,4 +50,8 @@ def make_environment(config):
|
||||||
if value is not None:
|
if value is not None:
|
||||||
environment[environment_variable_name] = 'YES' if value else 'NO'
|
environment[environment_variable_name] = 'YES' if value else 'NO'
|
||||||
|
|
||||||
|
# On Borg 1.4.0a1+, take advantage of more specific exit codes. No effect on
|
||||||
|
# older versions of Borg.
|
||||||
|
environment['BORG_EXIT_CODES'] = 'modern'
|
||||||
|
|
||||||
return environment
|
return environment
|
||||||
|
|
|
@ -65,6 +65,7 @@ def export_key(
|
||||||
full_command,
|
full_command,
|
||||||
output_file=output_file,
|
output_file=output_file,
|
||||||
output_log_level=logging.ANSWER,
|
output_log_level=logging.ANSWER,
|
||||||
borg_local_path=local_path,
|
|
||||||
extra_environment=environment.make_environment(config),
|
extra_environment=environment.make_environment(config),
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
|
|
|
@ -69,6 +69,7 @@ def export_tar_archive(
|
||||||
full_command,
|
full_command,
|
||||||
output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
|
output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
|
||||||
output_log_level=output_log_level,
|
output_log_level=output_log_level,
|
||||||
borg_local_path=local_path,
|
|
||||||
extra_environment=environment.make_environment(config),
|
extra_environment=environment.make_environment(config),
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
|
|
|
@ -57,7 +57,11 @@ def extract_last_archive_dry_run(
|
||||||
)
|
)
|
||||||
|
|
||||||
execute_command(
|
execute_command(
|
||||||
full_extract_command, working_directory=None, extra_environment=borg_environment
|
full_extract_command,
|
||||||
|
working_directory=None,
|
||||||
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -100,8 +104,13 @@ def extract_archive(
|
||||||
if not paths:
|
if not paths:
|
||||||
raise ValueError('The --strip-components flag with "all" requires at least one --path')
|
raise ValueError('The --strip-components flag with "all" requires at least one --path')
|
||||||
|
|
||||||
# Calculate the maximum number of leading path components of the given paths.
|
# Calculate the maximum number of leading path components of the given paths. "if piece"
|
||||||
strip_components = max(0, *(len(path.split(os.path.sep)) - 1 for path in paths))
|
# ignores empty path components, e.g. those resulting from a leading slash. And the "- 1"
|
||||||
|
# is so this doesn't count the final path component, e.g. the filename itself.
|
||||||
|
strip_components = max(
|
||||||
|
0,
|
||||||
|
*(len(tuple(piece for piece in path.split(os.path.sep) if piece)) - 1 for path in paths)
|
||||||
|
)
|
||||||
|
|
||||||
full_command = (
|
full_command = (
|
||||||
(local_path, 'extract')
|
(local_path, 'extract')
|
||||||
|
@ -127,6 +136,7 @@ def extract_archive(
|
||||||
)
|
)
|
||||||
|
|
||||||
borg_environment = environment.make_environment(config)
|
borg_environment = environment.make_environment(config)
|
||||||
|
borg_exit_codes = config.get('borg_exit_codes')
|
||||||
|
|
||||||
# The progress output isn't compatible with captured and logged output, as progress messes with
|
# The progress output isn't compatible with captured and logged output, as progress messes with
|
||||||
# the terminal directly.
|
# the terminal directly.
|
||||||
|
@ -136,6 +146,8 @@ def extract_archive(
|
||||||
output_file=DO_NOT_CAPTURE,
|
output_file=DO_NOT_CAPTURE,
|
||||||
working_directory=destination_path,
|
working_directory=destination_path,
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@ -146,10 +158,16 @@ def extract_archive(
|
||||||
working_directory=destination_path,
|
working_directory=destination_path,
|
||||||
run_to_completion=False,
|
run_to_completion=False,
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
|
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
|
||||||
# if the restore paths don't exist in the archive.
|
# if the restore paths don't exist in the archive.
|
||||||
execute_command(
|
execute_command(
|
||||||
full_command, working_directory=destination_path, extra_environment=borg_environment
|
full_command,
|
||||||
|
working_directory=destination_path,
|
||||||
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,8 +1,12 @@
|
||||||
import itertools
|
import itertools
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from borgmatic.borg import feature
|
from borgmatic.borg import feature
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def make_flags(name, value):
|
def make_flags(name, value):
|
||||||
'''
|
'''
|
||||||
|
@ -59,23 +63,28 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003
|
||||||
|
|
||||||
|
|
||||||
def make_match_archives_flags(match_archives, archive_name_format, local_borg_version):
|
def make_match_archives_flags(match_archives, archive_name_format, local_borg_version):
|
||||||
'''
|
'''
|
||||||
Return match archives flags based on the given match archives value, if any. If it isn't set,
|
Return match archives flags based on the given match archives value, if any. If it isn't set,
|
||||||
return match archives flags to match archives created with the given archive name format, if
|
return match archives flags to match archives created with the given (or default) archive name
|
||||||
any. This is done by replacing certain archive name format placeholders for ephemeral data (like
|
format. This is done by replacing certain archive name format placeholders for ephemeral data
|
||||||
"{now}") with globs.
|
(like "{now}") with globs.
|
||||||
'''
|
'''
|
||||||
if match_archives:
|
if match_archives:
|
||||||
|
if match_archives in {'*', 're:.*', 'sh:*'}:
|
||||||
|
return ()
|
||||||
|
|
||||||
if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
|
if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
|
||||||
return ('--match-archives', match_archives)
|
return ('--match-archives', match_archives)
|
||||||
else:
|
else:
|
||||||
return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
|
return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
|
||||||
|
|
||||||
if not archive_name_format:
|
derived_match_archives = re.sub(
|
||||||
return ()
|
r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format or DEFAULT_ARCHIVE_NAME_FORMAT
|
||||||
|
)
|
||||||
derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format)
|
|
||||||
|
|
||||||
if derived_match_archives == '*':
|
if derived_match_archives == '*':
|
||||||
return ()
|
return ()
|
||||||
|
@ -84,3 +93,26 @@ def make_match_archives_flags(match_archives, archive_name_format, local_borg_ve
|
||||||
return ('--match-archives', f'sh:{derived_match_archives}')
|
return ('--match-archives', f'sh:{derived_match_archives}')
|
||||||
else:
|
else:
|
||||||
return ('--glob-archives', f'{derived_match_archives}')
|
return ('--glob-archives', f'{derived_match_archives}')
|
||||||
|
|
||||||
|
|
||||||
|
def warn_for_aggressive_archive_flags(json_command, json_output):
|
||||||
|
'''
|
||||||
|
Given a JSON archives command and the resulting JSON string output from running it, parse the
|
||||||
|
JSON and warn if the command used an archive flag but the output indicates zero archives were
|
||||||
|
found.
|
||||||
|
'''
|
||||||
|
archive_flags_used = {'--glob-archives', '--match-archives'}.intersection(set(json_command))
|
||||||
|
|
||||||
|
if not archive_flags_used:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
if len(json.loads(json_output)['archives']) == 0:
|
||||||
|
logger.warning('An archive filter was applied, but no matching archives were found.')
|
||||||
|
logger.warning(
|
||||||
|
'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.'
|
||||||
|
)
|
||||||
|
except json.JSONDecodeError as error:
|
||||||
|
logger.debug(f'Cannot parse JSON output from archive command: {error}')
|
||||||
|
except (TypeError, KeyError):
|
||||||
|
logger.debug('Cannot parse JSON output from archive command: No "archives" key found')
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import borgmatic.logger
|
import borgmatic.logger
|
||||||
|
@ -7,24 +8,21 @@ from borgmatic.execute import execute_command, execute_command_and_capture_outpu
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def display_archives_info(
|
def make_info_command(
|
||||||
repository_path,
|
repository_path,
|
||||||
config,
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
info_arguments,
|
info_arguments,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
local_path='borg',
|
local_path,
|
||||||
remote_path=None,
|
remote_path,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given a local or remote repository path, a configuration dict, the local Borg version, global
|
Given a local or remote repository path, a configuration dict, the local Borg version, the
|
||||||
arguments as an argparse.Namespace, and the arguments to the info action, display summary
|
arguments to the info action as an argparse.Namespace, and global arguments, return a command
|
||||||
information for Borg archives in the repository or return JSON summary information.
|
as a tuple to display summary information for archives in the repository.
|
||||||
'''
|
'''
|
||||||
borgmatic.logger.add_custom_log_levels()
|
return (
|
||||||
lock_wait = config.get('lock_wait', None)
|
|
||||||
|
|
||||||
full_command = (
|
|
||||||
(local_path, 'info')
|
(local_path, 'info')
|
||||||
+ (
|
+ (
|
||||||
('--info',)
|
('--info',)
|
||||||
|
@ -38,7 +36,7 @@ def display_archives_info(
|
||||||
)
|
)
|
||||||
+ flags.make_flags('remote-path', remote_path)
|
+ flags.make_flags('remote-path', remote_path)
|
||||||
+ flags.make_flags('log-json', global_arguments.log_json)
|
+ flags.make_flags('log-json', global_arguments.log_json)
|
||||||
+ flags.make_flags('lock-wait', lock_wait)
|
+ flags.make_flags('lock-wait', config.get('lock_wait'))
|
||||||
+ (
|
+ (
|
||||||
(
|
(
|
||||||
flags.make_flags('match-archives', f'sh:{info_arguments.prefix}*')
|
flags.make_flags('match-archives', f'sh:{info_arguments.prefix}*')
|
||||||
|
@ -62,16 +60,59 @@ def display_archives_info(
|
||||||
+ flags.make_repository_flags(repository_path, local_borg_version)
|
+ flags.make_repository_flags(repository_path, local_borg_version)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def display_archives_info(
|
||||||
|
repository_path,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
info_arguments,
|
||||||
|
global_arguments,
|
||||||
|
local_path='borg',
|
||||||
|
remote_path=None,
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Given a local or remote repository path, a configuration dict, the local Borg version, the
|
||||||
|
arguments to the info action as an argparse.Namespace, and global arguments, display summary
|
||||||
|
information for Borg archives in the repository or return JSON summary information.
|
||||||
|
'''
|
||||||
|
borgmatic.logger.add_custom_log_levels()
|
||||||
|
|
||||||
|
main_command = make_info_command(
|
||||||
|
repository_path,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
info_arguments,
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
)
|
||||||
|
json_command = make_info_command(
|
||||||
|
repository_path,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
argparse.Namespace(**dict(info_arguments.__dict__, json=True)),
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
)
|
||||||
|
borg_exit_codes = config.get('borg_exit_codes')
|
||||||
|
|
||||||
|
json_info = execute_command_and_capture_output(
|
||||||
|
json_command,
|
||||||
|
extra_environment=environment.make_environment(config),
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
|
)
|
||||||
|
|
||||||
if info_arguments.json:
|
if info_arguments.json:
|
||||||
return execute_command_and_capture_output(
|
return json_info
|
||||||
full_command,
|
|
||||||
extra_environment=environment.make_environment(config),
|
flags.warn_for_aggressive_archive_flags(json_command, json_info)
|
||||||
borg_local_path=local_path,
|
|
||||||
)
|
execute_command(
|
||||||
else:
|
main_command,
|
||||||
execute_command(
|
output_log_level=logging.ANSWER,
|
||||||
full_command,
|
extra_environment=environment.make_environment(config),
|
||||||
output_log_level=logging.ANSWER,
|
borg_local_path=local_path,
|
||||||
borg_local_path=local_path,
|
borg_exit_codes=borg_exit_codes,
|
||||||
extra_environment=environment.make_environment(config),
|
)
|
||||||
)
|
|
||||||
|
|
|
@ -79,9 +79,11 @@ def make_find_paths(find_paths):
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
return tuple(
|
return tuple(
|
||||||
find_path
|
(
|
||||||
if re.compile(r'([-!+RrPp] )|(\w\w:)').match(find_path)
|
find_path
|
||||||
else f'sh:**/*{find_path}*/**'
|
if re.compile(r'([-!+RrPp] )|(\w\w:)').match(find_path)
|
||||||
|
else f'sh:**/*{find_path}*/**'
|
||||||
|
)
|
||||||
for find_path in find_paths
|
for find_path in find_paths
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -92,15 +94,16 @@ def capture_archive_listing(
|
||||||
config,
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
list_path=None,
|
list_paths=None,
|
||||||
|
path_format=None,
|
||||||
local_path='borg',
|
local_path='borg',
|
||||||
remote_path=None,
|
remote_path=None,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given a local or remote repository path, an archive name, a configuration dict, the local Borg
|
Given a local or remote repository path, an archive name, a configuration dict, the local Borg
|
||||||
version, global arguments as an argparse.Namespace, the archive path in which to list files, and
|
version, global arguments as an argparse.Namespace, the archive paths in which to list files,
|
||||||
local and remote Borg paths, capture the output of listing that archive and return it as a list
|
the Borg path format to use for the output, and local and remote Borg paths, capture the output
|
||||||
of file paths.
|
of listing that archive and return it as a list of file paths.
|
||||||
'''
|
'''
|
||||||
borg_environment = environment.make_environment(config)
|
borg_environment = environment.make_environment(config)
|
||||||
|
|
||||||
|
@ -113,10 +116,10 @@ def capture_archive_listing(
|
||||||
argparse.Namespace(
|
argparse.Namespace(
|
||||||
repository=repository_path,
|
repository=repository_path,
|
||||||
archive=archive,
|
archive=archive,
|
||||||
paths=[f'sh:{list_path}'],
|
paths=[f'sh:{path}' for path in list_paths] if list_paths else None,
|
||||||
find_paths=None,
|
find_paths=None,
|
||||||
json=None,
|
json=None,
|
||||||
format='{path}{NL}', # noqa: FS003
|
format=path_format or '{path}{NL}', # noqa: FS003
|
||||||
),
|
),
|
||||||
global_arguments,
|
global_arguments,
|
||||||
local_path,
|
local_path,
|
||||||
|
@ -124,6 +127,7 @@ def capture_archive_listing(
|
||||||
),
|
),
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
borg_local_path=local_path,
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
.strip('\n')
|
.strip('\n')
|
||||||
.split('\n')
|
.split('\n')
|
||||||
|
@ -189,6 +193,7 @@ def list_archive(
|
||||||
)
|
)
|
||||||
|
|
||||||
borg_environment = environment.make_environment(config)
|
borg_environment = environment.make_environment(config)
|
||||||
|
borg_exit_codes = config.get('borg_exit_codes')
|
||||||
|
|
||||||
# If there are any paths to find (and there's not a single archive already selected), start by
|
# If there are any paths to find (and there's not a single archive already selected), start by
|
||||||
# getting a list of archives to search.
|
# getting a list of archives to search.
|
||||||
|
@ -219,6 +224,7 @@ def list_archive(
|
||||||
),
|
),
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
borg_local_path=local_path,
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
.strip('\n')
|
.strip('\n')
|
||||||
.split('\n')
|
.split('\n')
|
||||||
|
@ -251,6 +257,7 @@ def list_archive(
|
||||||
execute_command(
|
execute_command(
|
||||||
main_command,
|
main_command,
|
||||||
output_log_level=logging.ANSWER,
|
output_log_level=logging.ANSWER,
|
||||||
borg_local_path=local_path,
|
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
|
|
|
@ -65,9 +65,15 @@ def mount_archive(
|
||||||
execute_command(
|
execute_command(
|
||||||
full_command,
|
full_command,
|
||||||
output_file=DO_NOT_CAPTURE,
|
output_file=DO_NOT_CAPTURE,
|
||||||
borg_local_path=local_path,
|
|
||||||
extra_environment=borg_environment,
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
execute_command(full_command, borg_local_path=local_path, extra_environment=borg_environment)
|
execute_command(
|
||||||
|
full_command,
|
||||||
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
|
)
|
||||||
|
|
|
@ -94,6 +94,7 @@ def prune_archives(
|
||||||
execute_command(
|
execute_command(
|
||||||
full_command,
|
full_command,
|
||||||
output_log_level=output_log_level,
|
output_log_level=output_log_level,
|
||||||
borg_local_path=local_path,
|
|
||||||
extra_environment=environment.make_environment(config),
|
extra_environment=environment.make_environment(config),
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import argparse
|
import argparse
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
@ -8,7 +9,7 @@ from borgmatic.execute import DO_NOT_CAPTURE, execute_command
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
|
RINFO_REPOSITORY_NOT_FOUND_EXIT_CODES = {2, 13}
|
||||||
|
|
||||||
|
|
||||||
def create_repository(
|
def create_repository(
|
||||||
|
@ -31,21 +32,34 @@ def create_repository(
|
||||||
version, a Borg encryption mode, the path to another repo whose key material should be reused,
|
version, a Borg encryption mode, the path to another repo whose key material should be reused,
|
||||||
whether the repository should be append-only, and the storage quota to use, create the
|
whether the repository should be append-only, and the storage quota to use, create the
|
||||||
repository. If the repository already exists, then log and skip creation.
|
repository. If the repository already exists, then log and skip creation.
|
||||||
|
|
||||||
|
Raise ValueError if the requested encryption mode does not match that of the repository.
|
||||||
|
Raise json.decoder.JSONDecodeError if the "borg info" JSON outputcannot be decoded.
|
||||||
|
Raise subprocess.CalledProcessError if "borg info" returns an error exit code.
|
||||||
'''
|
'''
|
||||||
try:
|
try:
|
||||||
rinfo.display_repository_info(
|
info_data = json.loads(
|
||||||
repository_path,
|
rinfo.display_repository_info(
|
||||||
config,
|
repository_path,
|
||||||
local_borg_version,
|
config,
|
||||||
argparse.Namespace(json=True),
|
local_borg_version,
|
||||||
global_arguments,
|
argparse.Namespace(json=True),
|
||||||
local_path,
|
global_arguments,
|
||||||
remote_path,
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
repository_encryption_mode = info_data.get('encryption', {}).get('mode')
|
||||||
|
|
||||||
|
if repository_encryption_mode != encryption_mode:
|
||||||
|
raise ValueError(
|
||||||
|
f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"'
|
||||||
|
)
|
||||||
|
|
||||||
logger.info(f'{repository_path}: Repository already exists. Skipping creation.')
|
logger.info(f'{repository_path}: Repository already exists. Skipping creation.')
|
||||||
return
|
return
|
||||||
except subprocess.CalledProcessError as error:
|
except subprocess.CalledProcessError as error:
|
||||||
if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
|
if error.returncode not in RINFO_REPOSITORY_NOT_FOUND_EXIT_CODES:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
lock_wait = config.get('lock_wait')
|
lock_wait = config.get('lock_wait')
|
||||||
|
@ -81,6 +95,7 @@ def create_repository(
|
||||||
execute_command(
|
execute_command(
|
||||||
rcreate_command,
|
rcreate_command,
|
||||||
output_file=DO_NOT_CAPTURE,
|
output_file=DO_NOT_CAPTURE,
|
||||||
borg_local_path=local_path,
|
|
||||||
extra_environment=environment.make_environment(config),
|
extra_environment=environment.make_environment(config),
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
|
|
|
@ -49,17 +49,20 @@ def display_repository_info(
|
||||||
)
|
)
|
||||||
|
|
||||||
extra_environment = environment.make_environment(config)
|
extra_environment = environment.make_environment(config)
|
||||||
|
borg_exit_codes = config.get('borg_exit_codes')
|
||||||
|
|
||||||
if rinfo_arguments.json:
|
if rinfo_arguments.json:
|
||||||
return execute_command_and_capture_output(
|
return execute_command_and_capture_output(
|
||||||
full_command,
|
full_command,
|
||||||
extra_environment=extra_environment,
|
extra_environment=extra_environment,
|
||||||
borg_local_path=local_path,
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
execute_command(
|
execute_command(
|
||||||
full_command,
|
full_command,
|
||||||
output_log_level=logging.ANSWER,
|
output_log_level=logging.ANSWER,
|
||||||
borg_local_path=local_path,
|
|
||||||
extra_environment=extra_environment,
|
extra_environment=extra_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import borgmatic.logger
|
import borgmatic.logger
|
||||||
|
@ -44,6 +45,7 @@ def resolve_archive_name(
|
||||||
full_command,
|
full_command,
|
||||||
extra_environment=environment.make_environment(config),
|
extra_environment=environment.make_environment(config),
|
||||||
borg_local_path=local_path,
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
latest_archive = output.strip().splitlines()[-1]
|
latest_archive = output.strip().splitlines()[-1]
|
||||||
|
@ -137,15 +139,33 @@ def list_repository(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
|
json_command = make_rlist_command(
|
||||||
|
repository_path,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
argparse.Namespace(**dict(rlist_arguments.__dict__, json=True)),
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
)
|
||||||
|
borg_exit_codes = config.get('borg_exit_codes')
|
||||||
|
|
||||||
|
json_listing = execute_command_and_capture_output(
|
||||||
|
json_command,
|
||||||
|
extra_environment=borg_environment,
|
||||||
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=borg_exit_codes,
|
||||||
|
)
|
||||||
|
|
||||||
if rlist_arguments.json:
|
if rlist_arguments.json:
|
||||||
return execute_command_and_capture_output(
|
return json_listing
|
||||||
main_command, extra_environment=borg_environment, borg_local_path=local_path
|
|
||||||
)
|
flags.warn_for_aggressive_archive_flags(json_command, json_listing)
|
||||||
else:
|
|
||||||
execute_command(
|
execute_command(
|
||||||
main_command,
|
main_command,
|
||||||
output_log_level=logging.ANSWER,
|
output_log_level=logging.ANSWER,
|
||||||
borg_local_path=local_path,
|
extra_environment=borg_environment,
|
||||||
extra_environment=borg_environment,
|
borg_local_path=local_path,
|
||||||
)
|
borg_exit_codes=borg_exit_codes,
|
||||||
|
)
|
||||||
|
|
|
@ -56,5 +56,6 @@ def transfer_archives(
|
||||||
output_log_level=logging.ANSWER,
|
output_log_level=logging.ANSWER,
|
||||||
output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
|
output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
|
||||||
borg_local_path=local_path,
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
extra_environment=environment.make_environment(config),
|
extra_environment=environment.make_environment(config),
|
||||||
)
|
)
|
||||||
|
|
|
@ -5,7 +5,7 @@ from borgmatic.execute import execute_command
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def unmount_archive(mount_point, local_path='borg'):
|
def unmount_archive(config, mount_point, local_path='borg'):
|
||||||
'''
|
'''
|
||||||
Given a mounted filesystem mount point, and an optional local Borg paths, umount the filesystem
|
Given a mounted filesystem mount point, and an optional local Borg paths, umount the filesystem
|
||||||
from the mount point.
|
from the mount point.
|
||||||
|
@ -17,4 +17,6 @@ def unmount_archive(mount_point, local_path='borg'):
|
||||||
+ (mount_point,)
|
+ (mount_point,)
|
||||||
)
|
)
|
||||||
|
|
||||||
execute_command(full_command)
|
execute_command(
|
||||||
|
full_command, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes')
|
||||||
|
)
|
||||||
|
|
|
@ -22,6 +22,7 @@ def local_borg_version(config, local_path='borg'):
|
||||||
full_command,
|
full_command,
|
||||||
extra_environment=environment.make_environment(config),
|
extra_environment=environment.make_environment(config),
|
||||||
borg_local_path=local_path,
|
borg_local_path=local_path,
|
||||||
|
borg_exit_codes=config.get('borg_exit_codes'),
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -259,28 +259,28 @@ def make_parsers():
|
||||||
type=int,
|
type=int,
|
||||||
choices=range(-2, 3),
|
choices=range(-2, 3),
|
||||||
default=0,
|
default=0,
|
||||||
help='Display verbose progress to the console (disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2)',
|
help='Display verbose progress to the console: -2 (disabled), -1 (errors only), 0 (responses to actions, the default), 1 (info about steps borgmatic is taking), or 2 (debug)',
|
||||||
)
|
)
|
||||||
global_group.add_argument(
|
global_group.add_argument(
|
||||||
'--syslog-verbosity',
|
'--syslog-verbosity',
|
||||||
type=int,
|
type=int,
|
||||||
choices=range(-2, 3),
|
choices=range(-2, 3),
|
||||||
default=0,
|
default=-2,
|
||||||
help='Log verbose progress to syslog (disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given',
|
help='Log verbose progress to syslog: -2 (disabled, the default), -1 (errors only), 0 (responses to actions), 1 (info about steps borgmatic is taking), or 2 (debug)',
|
||||||
)
|
)
|
||||||
global_group.add_argument(
|
global_group.add_argument(
|
||||||
'--log-file-verbosity',
|
'--log-file-verbosity',
|
||||||
type=int,
|
type=int,
|
||||||
choices=range(-2, 3),
|
choices=range(-2, 3),
|
||||||
default=0,
|
default=1,
|
||||||
help='Log verbose progress to log file (disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Only used when --log-file is given',
|
help='When --log-file is given, log verbose progress to file: -2 (disabled), -1 (errors only), 0 (responses to actions), 1 (info about steps borgmatic is taking, the default), or 2 (debug)',
|
||||||
)
|
)
|
||||||
global_group.add_argument(
|
global_group.add_argument(
|
||||||
'--monitoring-verbosity',
|
'--monitoring-verbosity',
|
||||||
type=int,
|
type=int,
|
||||||
choices=range(-2, 3),
|
choices=range(-2, 3),
|
||||||
default=0,
|
default=1,
|
||||||
help='Log verbose progress to monitoring integrations that support logging (from disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2)',
|
help='When a monitoring integration supporting logging is configured, log verbose progress to it: -2 (disabled), -1 (errors only), responses to actions (0), 1 (info about steps borgmatic is taking, the default), or 2 (debug)',
|
||||||
)
|
)
|
||||||
global_group.add_argument(
|
global_group.add_argument(
|
||||||
'--log-file',
|
'--log-file',
|
||||||
|
@ -302,7 +302,7 @@ def make_parsers():
|
||||||
metavar='OPTION.SUBOPTION=VALUE',
|
metavar='OPTION.SUBOPTION=VALUE',
|
||||||
dest='overrides',
|
dest='overrides',
|
||||||
action='append',
|
action='append',
|
||||||
help='Configuration file option to override with specified value, can specify flag multiple times',
|
help='Configuration file option to override with specified value, see documentation for overriding list or key/value options, can specify flag multiple times',
|
||||||
)
|
)
|
||||||
global_group.add_argument(
|
global_group.add_argument(
|
||||||
'--no-environment-interpolation',
|
'--no-environment-interpolation',
|
||||||
|
@ -467,8 +467,8 @@ def make_parsers():
|
||||||
prune_parser = action_parsers.add_parser(
|
prune_parser = action_parsers.add_parser(
|
||||||
'prune',
|
'prune',
|
||||||
aliases=ACTION_ALIASES['prune'],
|
aliases=ACTION_ALIASES['prune'],
|
||||||
help='Prune archives according to the retention policy (with Borg 1.2+, run compact afterwards to actually free space)',
|
help='Prune archives according to the retention policy (with Borg 1.2+, you must run compact afterwards to actually free space)',
|
||||||
description='Prune archives according to the retention policy (with Borg 1.2+, run compact afterwards to actually free space)',
|
description='Prune archives according to the retention policy (with Borg 1.2+, you must run compact afterwards to actually free space)',
|
||||||
add_help=False,
|
add_help=False,
|
||||||
)
|
)
|
||||||
prune_group = prune_parser.add_argument_group('prune arguments')
|
prune_group = prune_parser.add_argument_group('prune arguments')
|
||||||
|
@ -604,13 +604,20 @@ def make_parsers():
|
||||||
action='store_true',
|
action='store_true',
|
||||||
help='Attempt to repair any inconsistencies found (for interactive use)',
|
help='Attempt to repair any inconsistencies found (for interactive use)',
|
||||||
)
|
)
|
||||||
|
check_group.add_argument(
|
||||||
|
'-a',
|
||||||
|
'--match-archives',
|
||||||
|
'--glob-archives',
|
||||||
|
metavar='PATTERN',
|
||||||
|
help='Only check archives with names matching this pattern',
|
||||||
|
)
|
||||||
check_group.add_argument(
|
check_group.add_argument(
|
||||||
'--only',
|
'--only',
|
||||||
metavar='CHECK',
|
metavar='CHECK',
|
||||||
choices=('repository', 'archives', 'data', 'extract'),
|
choices=('repository', 'archives', 'data', 'extract', 'spot'),
|
||||||
dest='only',
|
dest='only_checks',
|
||||||
action='append',
|
action='append',
|
||||||
help='Run a particular consistency check (repository, archives, data, or extract) instead of configured checks (subject to configured frequency, can specify flag multiple times)',
|
help='Run a particular consistency check (repository, archives, data, extract, or spot) instead of configured checks (subject to configured frequency, can specify flag multiple times)',
|
||||||
)
|
)
|
||||||
check_group.add_argument(
|
check_group.add_argument(
|
||||||
'--force',
|
'--force',
|
||||||
|
@ -724,6 +731,11 @@ def make_parsers():
|
||||||
action='store_true',
|
action='store_true',
|
||||||
help='Display progress for each file as it is extracted',
|
help='Display progress for each file as it is extracted',
|
||||||
)
|
)
|
||||||
|
config_bootstrap_group.add_argument(
|
||||||
|
'--ssh-command',
|
||||||
|
metavar='COMMAND',
|
||||||
|
help='Command to use instead of "ssh"',
|
||||||
|
)
|
||||||
config_bootstrap_group.add_argument(
|
config_bootstrap_group.add_argument(
|
||||||
'-h', '--help', action='help', help='Show this help message and exit'
|
'-h', '--help', action='help', help='Show this help message and exit'
|
||||||
)
|
)
|
||||||
|
@ -906,8 +918,8 @@ def make_parsers():
|
||||||
restore_parser = action_parsers.add_parser(
|
restore_parser = action_parsers.add_parser(
|
||||||
'restore',
|
'restore',
|
||||||
aliases=ACTION_ALIASES['restore'],
|
aliases=ACTION_ALIASES['restore'],
|
||||||
help='Restore database dumps from a named archive',
|
help='Restore data source (e.g. database) dumps from a named archive',
|
||||||
description='Restore database dumps from a named archive. (To extract files instead, use "borgmatic extract".)',
|
description='Restore data source (e.g. database) dumps from a named archive. (To extract files instead, use "borgmatic extract".)',
|
||||||
add_help=False,
|
add_help=False,
|
||||||
)
|
)
|
||||||
restore_group = restore_parser.add_argument_group('restore arguments')
|
restore_group = restore_parser.add_argument_group('restore arguments')
|
||||||
|
@ -919,18 +931,19 @@ def make_parsers():
|
||||||
'--archive', help='Name of archive to restore from (or "latest")', required=True
|
'--archive', help='Name of archive to restore from (or "latest")', required=True
|
||||||
)
|
)
|
||||||
restore_group.add_argument(
|
restore_group.add_argument(
|
||||||
|
'--data-source',
|
||||||
'--database',
|
'--database',
|
||||||
metavar='NAME',
|
metavar='NAME',
|
||||||
dest='databases',
|
dest='data_sources',
|
||||||
action='append',
|
action='append',
|
||||||
help="Name of database to restore from archive, must be defined in borgmatic's configuration, can specify flag multiple times, defaults to all databases",
|
help="Name of data source (e.g. database) to restore from archive, must be defined in borgmatic's configuration, can specify flag multiple times, defaults to all data sources in the archive",
|
||||||
)
|
)
|
||||||
restore_group.add_argument(
|
restore_group.add_argument(
|
||||||
'--schema',
|
'--schema',
|
||||||
metavar='NAME',
|
metavar='NAME',
|
||||||
dest='schemas',
|
dest='schemas',
|
||||||
action='append',
|
action='append',
|
||||||
help='Name of schema to restore from the database, can specify flag multiple times, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases',
|
help='Name of schema to restore from the data source, can specify flag multiple times, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases',
|
||||||
)
|
)
|
||||||
restore_group.add_argument(
|
restore_group.add_argument(
|
||||||
'--hostname',
|
'--hostname',
|
||||||
|
@ -938,7 +951,7 @@ def make_parsers():
|
||||||
)
|
)
|
||||||
restore_group.add_argument(
|
restore_group.add_argument(
|
||||||
'--port',
|
'--port',
|
||||||
help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
|
help='Database port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
|
||||||
)
|
)
|
||||||
restore_group.add_argument(
|
restore_group.add_argument(
|
||||||
'--username',
|
'--username',
|
||||||
|
@ -1327,4 +1340,7 @@ def parse_arguments(*unparsed_arguments):
|
||||||
'With the info action, only one of --archive, --prefix, or --match-archives flags can be used.'
|
'With the info action, only one of --archive, --prefix, or --match-archives flags can be used.'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if 'borg' in arguments and arguments['global'].dry_run:
|
||||||
|
raise ValueError('With the borg action, --dry-run is not supported.')
|
||||||
|
|
||||||
return arguments
|
return arguments
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import collections
|
import collections
|
||||||
|
import importlib.metadata
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
@ -9,11 +10,6 @@ from subprocess import CalledProcessError
|
||||||
|
|
||||||
import colorama
|
import colorama
|
||||||
|
|
||||||
try:
|
|
||||||
import importlib_metadata
|
|
||||||
except ModuleNotFoundError: # pragma: nocover
|
|
||||||
import importlib.metadata as importlib_metadata
|
|
||||||
|
|
||||||
import borgmatic.actions.borg
|
import borgmatic.actions.borg
|
||||||
import borgmatic.actions.break_lock
|
import borgmatic.actions.break_lock
|
||||||
import borgmatic.actions.check
|
import borgmatic.actions.check
|
||||||
|
@ -48,11 +44,25 @@ from borgmatic.verbosity import verbosity_to_log_level
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def run_configuration(config_filename, config, arguments):
|
def get_skip_actions(config, arguments):
|
||||||
'''
|
'''
|
||||||
Given a config filename, the corresponding parsed config dict, and command-line arguments as a
|
Given a configuration dict and command-line arguments as an argparse.Namespace, return a list of
|
||||||
dict from subparser name to a namespace of parsed arguments, execute the defined create, prune,
|
the configured action names to skip. Omit "check" from this list though if "check --force" is
|
||||||
compact, check, and/or other actions.
|
part of the command-like arguments.
|
||||||
|
'''
|
||||||
|
skip_actions = config.get('skip_actions', [])
|
||||||
|
|
||||||
|
if 'check' in arguments and arguments['check'].force:
|
||||||
|
return [action for action in skip_actions if action != 'check']
|
||||||
|
|
||||||
|
return skip_actions
|
||||||
|
|
||||||
|
|
||||||
|
def run_configuration(config_filename, config, config_paths, arguments):
|
||||||
|
'''
|
||||||
|
Given a config filename, the corresponding parsed config dict, a sequence of loaded
|
||||||
|
configuration paths, and command-line arguments as a dict from subparser name to a namespace of
|
||||||
|
parsed arguments, execute the defined create, prune, compact, check, and/or other actions.
|
||||||
|
|
||||||
Yield a combination of:
|
Yield a combination of:
|
||||||
|
|
||||||
|
@ -70,9 +80,16 @@ def run_configuration(config_filename, config, arguments):
|
||||||
using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments)
|
using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments)
|
||||||
monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity)
|
monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity)
|
||||||
monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED
|
monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED
|
||||||
|
skip_actions = get_skip_actions(config, arguments)
|
||||||
|
|
||||||
|
if skip_actions:
|
||||||
|
logger.debug(
|
||||||
|
f"{config_filename}: Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions"
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
local_borg_version = borg_version.local_borg_version(config, local_path)
|
local_borg_version = borg_version.local_borg_version(config, local_path)
|
||||||
|
logger.debug(f'{config_filename}: Borg {local_borg_version}')
|
||||||
except (OSError, CalledProcessError, ValueError) as error:
|
except (OSError, CalledProcessError, ValueError) as error:
|
||||||
yield from log_error_records(f'{config_filename}: Error getting local Borg version', error)
|
yield from log_error_records(f'{config_filename}: Error getting local Borg version', error)
|
||||||
return
|
return
|
||||||
|
@ -127,6 +144,7 @@ def run_configuration(config_filename, config, arguments):
|
||||||
arguments=arguments,
|
arguments=arguments,
|
||||||
config_filename=config_filename,
|
config_filename=config_filename,
|
||||||
config=config,
|
config=config,
|
||||||
|
config_paths=config_paths,
|
||||||
local_path=local_path,
|
local_path=local_path,
|
||||||
remote_path=remote_path,
|
remote_path=remote_path,
|
||||||
local_borg_version=local_borg_version,
|
local_borg_version=local_borg_version,
|
||||||
|
@ -151,7 +169,7 @@ def run_configuration(config_filename, config, arguments):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if command.considered_soft_failure(config_filename, error):
|
if command.considered_soft_failure(config_filename, error):
|
||||||
return
|
break
|
||||||
|
|
||||||
yield from log_error_records(
|
yield from log_error_records(
|
||||||
f'{repository.get("label", repository["path"])}: Error running actions for repository',
|
f'{repository.get("label", repository["path"])}: Error running actions for repository',
|
||||||
|
@ -162,7 +180,7 @@ def run_configuration(config_filename, config, arguments):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if monitoring_hooks_are_activated:
|
if monitoring_hooks_are_activated:
|
||||||
# send logs irrespective of error
|
# Send logs irrespective of error.
|
||||||
dispatch.call_hooks(
|
dispatch.call_hooks(
|
||||||
'ping_monitor',
|
'ping_monitor',
|
||||||
config,
|
config,
|
||||||
|
@ -173,11 +191,9 @@ def run_configuration(config_filename, config, arguments):
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
except (OSError, CalledProcessError) as error:
|
except (OSError, CalledProcessError) as error:
|
||||||
if command.considered_soft_failure(config_filename, error):
|
if not command.considered_soft_failure(config_filename, error):
|
||||||
return
|
encountered_error = error
|
||||||
|
yield from log_error_records(f'{repository["path"]}: Error pinging monitor', error)
|
||||||
encountered_error = error
|
|
||||||
yield from log_error_records(f'{repository["path"]}: Error pinging monitor', error)
|
|
||||||
|
|
||||||
if not encountered_error:
|
if not encountered_error:
|
||||||
try:
|
try:
|
||||||
|
@ -247,6 +263,7 @@ def run_actions(
|
||||||
arguments,
|
arguments,
|
||||||
config_filename,
|
config_filename,
|
||||||
config,
|
config,
|
||||||
|
config_paths,
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
|
@ -254,9 +271,9 @@ def run_actions(
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration
|
Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration
|
||||||
filename, several different configuration dicts, local and remote paths to Borg, a local Borg
|
filename, a configuration dict, a sequence of loaded configuration paths, local and remote paths
|
||||||
version string, and a repository name, run all actions from the command-line arguments on the
|
to Borg, a local Borg version string, and a repository name, run all actions from the
|
||||||
given repository.
|
command-line arguments on the given repository.
|
||||||
|
|
||||||
Yield JSON output strings from executing any actions that produce JSON.
|
Yield JSON output strings from executing any actions that produce JSON.
|
||||||
|
|
||||||
|
@ -274,6 +291,7 @@ def run_actions(
|
||||||
'repositories': ','.join([repo['path'] for repo in config['repositories']]),
|
'repositories': ','.join([repo['path'] for repo in config['repositories']]),
|
||||||
'log_file': global_arguments.log_file if global_arguments.log_file else '',
|
'log_file': global_arguments.log_file if global_arguments.log_file else '',
|
||||||
}
|
}
|
||||||
|
skip_actions = set(get_skip_actions(config, arguments))
|
||||||
|
|
||||||
command.execute_hook(
|
command.execute_hook(
|
||||||
config.get('before_actions'),
|
config.get('before_actions'),
|
||||||
|
@ -285,7 +303,7 @@ def run_actions(
|
||||||
)
|
)
|
||||||
|
|
||||||
for action_name, action_arguments in arguments.items():
|
for action_name, action_arguments in arguments.items():
|
||||||
if action_name == 'rcreate':
|
if action_name == 'rcreate' and action_name not in skip_actions:
|
||||||
borgmatic.actions.rcreate.run_rcreate(
|
borgmatic.actions.rcreate.run_rcreate(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -295,7 +313,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'transfer':
|
elif action_name == 'transfer' and action_name not in skip_actions:
|
||||||
borgmatic.actions.transfer.run_transfer(
|
borgmatic.actions.transfer.run_transfer(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -305,11 +323,12 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'create':
|
elif action_name == 'create' and action_name not in skip_actions:
|
||||||
yield from borgmatic.actions.create.run_create(
|
yield from borgmatic.actions.create.run_create(
|
||||||
config_filename,
|
config_filename,
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
config_paths,
|
||||||
hook_context,
|
hook_context,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
action_arguments,
|
action_arguments,
|
||||||
|
@ -318,7 +337,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'prune':
|
elif action_name == 'prune' and action_name not in skip_actions:
|
||||||
borgmatic.actions.prune.run_prune(
|
borgmatic.actions.prune.run_prune(
|
||||||
config_filename,
|
config_filename,
|
||||||
repository,
|
repository,
|
||||||
|
@ -331,7 +350,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'compact':
|
elif action_name == 'compact' and action_name not in skip_actions:
|
||||||
borgmatic.actions.compact.run_compact(
|
borgmatic.actions.compact.run_compact(
|
||||||
config_filename,
|
config_filename,
|
||||||
repository,
|
repository,
|
||||||
|
@ -344,7 +363,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'check':
|
elif action_name == 'check' and action_name not in skip_actions:
|
||||||
if checks.repository_enabled_for_checks(repository, config):
|
if checks.repository_enabled_for_checks(repository, config):
|
||||||
borgmatic.actions.check.run_check(
|
borgmatic.actions.check.run_check(
|
||||||
config_filename,
|
config_filename,
|
||||||
|
@ -357,7 +376,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'extract':
|
elif action_name == 'extract' and action_name not in skip_actions:
|
||||||
borgmatic.actions.extract.run_extract(
|
borgmatic.actions.extract.run_extract(
|
||||||
config_filename,
|
config_filename,
|
||||||
repository,
|
repository,
|
||||||
|
@ -369,7 +388,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'export-tar':
|
elif action_name == 'export-tar' and action_name not in skip_actions:
|
||||||
borgmatic.actions.export_tar.run_export_tar(
|
borgmatic.actions.export_tar.run_export_tar(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -379,7 +398,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'mount':
|
elif action_name == 'mount' and action_name not in skip_actions:
|
||||||
borgmatic.actions.mount.run_mount(
|
borgmatic.actions.mount.run_mount(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -389,7 +408,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'restore':
|
elif action_name == 'restore' and action_name not in skip_actions:
|
||||||
borgmatic.actions.restore.run_restore(
|
borgmatic.actions.restore.run_restore(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -399,7 +418,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'rlist':
|
elif action_name == 'rlist' and action_name not in skip_actions:
|
||||||
yield from borgmatic.actions.rlist.run_rlist(
|
yield from borgmatic.actions.rlist.run_rlist(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -409,7 +428,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'list':
|
elif action_name == 'list' and action_name not in skip_actions:
|
||||||
yield from borgmatic.actions.list.run_list(
|
yield from borgmatic.actions.list.run_list(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -419,7 +438,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'rinfo':
|
elif action_name == 'rinfo' and action_name not in skip_actions:
|
||||||
yield from borgmatic.actions.rinfo.run_rinfo(
|
yield from borgmatic.actions.rinfo.run_rinfo(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -429,7 +448,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'info':
|
elif action_name == 'info' and action_name not in skip_actions:
|
||||||
yield from borgmatic.actions.info.run_info(
|
yield from borgmatic.actions.info.run_info(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -439,7 +458,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'break-lock':
|
elif action_name == 'break-lock' and action_name not in skip_actions:
|
||||||
borgmatic.actions.break_lock.run_break_lock(
|
borgmatic.actions.break_lock.run_break_lock(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -449,7 +468,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'export':
|
elif action_name == 'export' and action_name not in skip_actions:
|
||||||
borgmatic.actions.export_key.run_export_key(
|
borgmatic.actions.export_key.run_export_key(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -459,7 +478,7 @@ def run_actions(
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
)
|
)
|
||||||
elif action_name == 'borg':
|
elif action_name == 'borg' and action_name not in skip_actions:
|
||||||
borgmatic.actions.borg.run_borg(
|
borgmatic.actions.borg.run_borg(
|
||||||
repository,
|
repository,
|
||||||
config,
|
config,
|
||||||
|
@ -484,13 +503,15 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
|
||||||
'''
|
'''
|
||||||
Given a sequence of configuration filenames, load and validate each configuration file. Return
|
Given a sequence of configuration filenames, load and validate each configuration file. Return
|
||||||
the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
|
the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
|
||||||
and sequence of logging.LogRecord instances containing any parse errors.
|
a sequence of paths for all loaded configuration files (including includes), and a sequence of
|
||||||
|
logging.LogRecord instances containing any parse errors.
|
||||||
|
|
||||||
Log records are returned here instead of being logged directly because logging isn't yet
|
Log records are returned here instead of being logged directly because logging isn't yet
|
||||||
initialized at this point!
|
initialized at this point!
|
||||||
'''
|
'''
|
||||||
# Dict mapping from config filename to corresponding parsed config dict.
|
# Dict mapping from config filename to corresponding parsed config dict.
|
||||||
configs = collections.OrderedDict()
|
configs = collections.OrderedDict()
|
||||||
|
config_paths = set()
|
||||||
logs = []
|
logs = []
|
||||||
|
|
||||||
# Parse and load each configuration file.
|
# Parse and load each configuration file.
|
||||||
|
@ -507,9 +528,10 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
configs[config_filename], parse_logs = validate.parse_configuration(
|
configs[config_filename], paths, parse_logs = validate.parse_configuration(
|
||||||
config_filename, validate.schema_filename(), overrides, resolve_env
|
config_filename, validate.schema_filename(), overrides, resolve_env
|
||||||
)
|
)
|
||||||
|
config_paths.update(paths)
|
||||||
logs.extend(parse_logs)
|
logs.extend(parse_logs)
|
||||||
except PermissionError:
|
except PermissionError:
|
||||||
logs.extend(
|
logs.extend(
|
||||||
|
@ -539,7 +561,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
return (configs, logs)
|
return (configs, sorted(config_paths), logs)
|
||||||
|
|
||||||
|
|
||||||
def log_record(suppress_log=False, **kwargs):
|
def log_record(suppress_log=False, **kwargs):
|
||||||
|
@ -555,9 +577,6 @@ def log_record(suppress_log=False, **kwargs):
|
||||||
return record
|
return record
|
||||||
|
|
||||||
|
|
||||||
MAX_CAPTURED_OUTPUT_LENGTH = 1000
|
|
||||||
|
|
||||||
|
|
||||||
def log_error_records(
|
def log_error_records(
|
||||||
message, error=None, levelno=logging.CRITICAL, log_command_error_output=False
|
message, error=None, levelno=logging.CRITICAL, log_command_error_output=False
|
||||||
):
|
):
|
||||||
|
@ -579,20 +598,24 @@ def log_error_records(
|
||||||
raise error
|
raise error
|
||||||
except CalledProcessError as error:
|
except CalledProcessError as error:
|
||||||
yield log_record(levelno=levelno, levelname=level_name, msg=message)
|
yield log_record(levelno=levelno, levelname=level_name, msg=message)
|
||||||
|
|
||||||
if error.output:
|
if error.output:
|
||||||
try:
|
try:
|
||||||
output = error.output.decode('utf-8')
|
output = error.output.decode('utf-8')
|
||||||
except (UnicodeDecodeError, AttributeError):
|
except (UnicodeDecodeError, AttributeError):
|
||||||
output = error.output
|
output = error.output
|
||||||
|
|
||||||
# Suppress these logs for now and save full error output for the log summary at the end.
|
# Suppress these logs for now and save the error output for the log summary at the end.
|
||||||
yield log_record(
|
# Log a separate record per line, as some errors can be really verbose and overflow the
|
||||||
levelno=levelno,
|
# per-record size limits imposed by some logging backends.
|
||||||
levelname=level_name,
|
for output_line in output.splitlines():
|
||||||
msg=output[:MAX_CAPTURED_OUTPUT_LENGTH]
|
yield log_record(
|
||||||
+ ' ...' * (len(output) > MAX_CAPTURED_OUTPUT_LENGTH),
|
levelno=levelno,
|
||||||
suppress_log=True,
|
levelname=level_name,
|
||||||
)
|
msg=output_line,
|
||||||
|
suppress_log=True,
|
||||||
|
)
|
||||||
|
|
||||||
yield log_record(levelno=levelno, levelname=level_name, msg=error)
|
yield log_record(levelno=levelno, levelname=level_name, msg=error)
|
||||||
except (ValueError, OSError) as error:
|
except (ValueError, OSError) as error:
|
||||||
yield log_record(levelno=levelno, levelname=level_name, msg=message)
|
yield log_record(levelno=levelno, levelname=level_name, msg=message)
|
||||||
|
@ -705,12 +728,12 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
def collect_configuration_run_summary_logs(configs, arguments):
|
def collect_configuration_run_summary_logs(configs, config_paths, arguments):
|
||||||
'''
|
'''
|
||||||
Given a dict of configuration filename to corresponding parsed configuration and parsed
|
Given a dict of configuration filename to corresponding parsed configuration, a sequence of
|
||||||
command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
|
loaded configuration paths, and parsed command-line arguments as a dict from subparser name to a
|
||||||
each configuration file and yield a series of logging.LogRecord instances containing summary
|
parsed namespace of arguments, run each configuration file and yield a series of
|
||||||
information about each run.
|
logging.LogRecord instances containing summary information about each run.
|
||||||
|
|
||||||
As a side effect of running through these configuration files, output their JSON results, if
|
As a side effect of running through these configuration files, output their JSON results, if
|
||||||
any, to stdout.
|
any, to stdout.
|
||||||
|
@ -755,7 +778,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
|
||||||
# Execute the actions corresponding to each configuration file.
|
# Execute the actions corresponding to each configuration file.
|
||||||
json_results = []
|
json_results = []
|
||||||
for config_filename, config in configs.items():
|
for config_filename, config in configs.items():
|
||||||
results = list(run_configuration(config_filename, config, arguments))
|
results = list(run_configuration(config_filename, config, config_paths, arguments))
|
||||||
error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord))
|
error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord))
|
||||||
|
|
||||||
if error_logs:
|
if error_logs:
|
||||||
|
@ -776,6 +799,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
|
||||||
logger.info(f"Unmounting mount point {arguments['umount'].mount_point}")
|
logger.info(f"Unmounting mount point {arguments['umount'].mount_point}")
|
||||||
try:
|
try:
|
||||||
borg_umount.unmount_archive(
|
borg_umount.unmount_archive(
|
||||||
|
config,
|
||||||
mount_point=arguments['umount'].mount_point,
|
mount_point=arguments['umount'].mount_point,
|
||||||
local_path=get_local_path(configs),
|
local_path=get_local_path(configs),
|
||||||
)
|
)
|
||||||
|
@ -826,7 +850,7 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||||
|
|
||||||
global_arguments = arguments['global']
|
global_arguments = arguments['global']
|
||||||
if global_arguments.version:
|
if global_arguments.version:
|
||||||
print(importlib_metadata.version('borgmatic'))
|
print(importlib.metadata.version('borgmatic'))
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
if global_arguments.bash_completion:
|
if global_arguments.bash_completion:
|
||||||
print(borgmatic.commands.completion.bash.bash_completion())
|
print(borgmatic.commands.completion.bash.bash_completion())
|
||||||
|
@ -836,8 +860,7 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
|
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
|
||||||
global_arguments.used_config_paths = list(config_filenames)
|
configs, config_paths, parse_logs = load_configurations(
|
||||||
configs, parse_logs = load_configurations(
|
|
||||||
config_filenames, global_arguments.overrides, global_arguments.resolve_env
|
config_filenames, global_arguments.overrides, global_arguments.resolve_env
|
||||||
)
|
)
|
||||||
configuration_parse_errors = (
|
configuration_parse_errors = (
|
||||||
|
@ -847,10 +870,8 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||||
any_json_flags = any(
|
any_json_flags = any(
|
||||||
getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()
|
getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()
|
||||||
)
|
)
|
||||||
colorama.init(
|
color_enabled = should_do_markup(global_arguments.no_color or any_json_flags, configs)
|
||||||
autoreset=True,
|
colorama.init(autoreset=color_enabled, strip=not color_enabled)
|
||||||
strip=not should_do_markup(global_arguments.no_color or any_json_flags, configs),
|
|
||||||
)
|
|
||||||
try:
|
try:
|
||||||
configure_logging(
|
configure_logging(
|
||||||
verbosity_to_log_level(global_arguments.verbosity),
|
verbosity_to_log_level(global_arguments.verbosity),
|
||||||
|
@ -859,6 +880,7 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||||
verbosity_to_log_level(global_arguments.monitoring_verbosity),
|
verbosity_to_log_level(global_arguments.monitoring_verbosity),
|
||||||
global_arguments.log_file,
|
global_arguments.log_file,
|
||||||
global_arguments.log_file_format,
|
global_arguments.log_file_format,
|
||||||
|
color_enabled=color_enabled,
|
||||||
)
|
)
|
||||||
except (FileNotFoundError, PermissionError) as error:
|
except (FileNotFoundError, PermissionError) as error:
|
||||||
configure_logging(logging.CRITICAL)
|
configure_logging(logging.CRITICAL)
|
||||||
|
@ -874,7 +896,7 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||||
configs, arguments, configuration_parse_errors
|
configs, arguments, configuration_parse_errors
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
or list(collect_configuration_run_summary_logs(configs, arguments))
|
or list(collect_configuration_run_summary_logs(configs, config_paths, arguments))
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
summary_logs_max_level = max(log.levelno for log in summary_logs)
|
summary_logs_max_level = max(log.levelno for log in summary_logs)
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
def repository_enabled_for_checks(repository, consistency):
|
def repository_enabled_for_checks(repository, config):
|
||||||
'''
|
'''
|
||||||
Given a repository name and a consistency configuration dict, return whether the repository
|
Given a repository name and a configuration dict, return whether the
|
||||||
is enabled to have consistency checks run.
|
repository is enabled to have consistency checks run.
|
||||||
'''
|
'''
|
||||||
if not consistency.get('check_repositories'):
|
if not config.get('check_repositories'):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
return repository in consistency['check_repositories']
|
return repository in config['check_repositories']
|
||||||
|
|
|
@ -0,0 +1,64 @@
|
||||||
|
import shlex
|
||||||
|
|
||||||
|
|
||||||
|
def coerce_scalar(value):
|
||||||
|
'''
|
||||||
|
Given a configuration value, coerce it to an integer or a boolean as appropriate and return the
|
||||||
|
result.
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
return int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
if value == 'true' or value == 'True':
|
||||||
|
return True
|
||||||
|
if value == 'false' or value == 'False':
|
||||||
|
return False
|
||||||
|
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def apply_constants(value, constants, shell_escape=False):
|
||||||
|
'''
|
||||||
|
Given a configuration value (bool, dict, int, list, or string) and a dict of named constants,
|
||||||
|
replace any configuration string values of the form "{constant}" (or containing it) with the
|
||||||
|
value of the correspondingly named key from the constants. Recurse as necessary into nested
|
||||||
|
configuration to find values to replace.
|
||||||
|
|
||||||
|
For instance, if a configuration value contains "{foo}", replace it with the value of the "foo"
|
||||||
|
key found within the configuration's "constants".
|
||||||
|
|
||||||
|
If shell escape is True, then escape the constant's value before applying it.
|
||||||
|
|
||||||
|
Return the configuration value and modify the original.
|
||||||
|
'''
|
||||||
|
if not value or not constants:
|
||||||
|
return value
|
||||||
|
|
||||||
|
if isinstance(value, str):
|
||||||
|
for constant_name, constant_value in constants.items():
|
||||||
|
value = value.replace(
|
||||||
|
'{' + constant_name + '}',
|
||||||
|
shlex.quote(str(constant_value)) if shell_escape else str(constant_value),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Support constants within non-string scalars by coercing the value to its appropriate type.
|
||||||
|
value = coerce_scalar(value)
|
||||||
|
elif isinstance(value, list):
|
||||||
|
for index, list_value in enumerate(value):
|
||||||
|
value[index] = apply_constants(list_value, constants, shell_escape)
|
||||||
|
elif isinstance(value, dict):
|
||||||
|
for option_name, option_value in value.items():
|
||||||
|
value[option_name] = apply_constants(
|
||||||
|
option_value,
|
||||||
|
constants,
|
||||||
|
shell_escape=(
|
||||||
|
shell_escape
|
||||||
|
or option_name.startswith('before_')
|
||||||
|
or option_name.startswith('after_')
|
||||||
|
or option_name == 'on_error'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
return value
|
|
@ -1,21 +1,22 @@
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
_VARIABLE_PATTERN = re.compile(
|
VARIABLE_PATTERN = re.compile(
|
||||||
r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})'
|
r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _resolve_string(matcher):
|
def resolve_string(matcher):
|
||||||
'''
|
'''
|
||||||
Get the value from environment given a matcher containing a name and an optional default value.
|
Given a matcher containing a name and an optional default value, get the value from environment.
|
||||||
If the variable is not defined in environment and no default value is provided, an Error is raised.
|
|
||||||
|
Raise ValueError if the variable is not defined in environment and no default value is provided.
|
||||||
'''
|
'''
|
||||||
if matcher.group('escape') is not None:
|
if matcher.group('escape') is not None:
|
||||||
# in case of escaped envvar, unescape it
|
# In the case of an escaped environment variable, unescape it.
|
||||||
return matcher.group('variable')
|
return matcher.group('variable')
|
||||||
|
|
||||||
# resolve the env var
|
# Resolve the environment variable.
|
||||||
name, default = matcher.group('name'), matcher.group('default')
|
name, default = matcher.group('name'), matcher.group('default')
|
||||||
out = os.getenv(name, default=default)
|
out = os.getenv(name, default=default)
|
||||||
|
|
||||||
|
@ -27,19 +28,24 @@ def _resolve_string(matcher):
|
||||||
|
|
||||||
def resolve_env_variables(item):
|
def resolve_env_variables(item):
|
||||||
'''
|
'''
|
||||||
Resolves variables like or ${FOO} from given configuration with values from process environment
|
Resolves variables like or ${FOO} from given configuration with values from process environment.
|
||||||
Supported formats:
|
|
||||||
- ${FOO} will return FOO env variable
|
|
||||||
- ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
|
|
||||||
|
|
||||||
If any variable is missing in environment and no default value is provided, an Error is raised.
|
Supported formats:
|
||||||
|
|
||||||
|
* ${FOO} will return FOO env variable
|
||||||
|
* ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
|
||||||
|
|
||||||
|
Raise if any variable is missing in environment and no default value is provided.
|
||||||
'''
|
'''
|
||||||
if isinstance(item, str):
|
if isinstance(item, str):
|
||||||
return _VARIABLE_PATTERN.sub(_resolve_string, item)
|
return VARIABLE_PATTERN.sub(resolve_string, item)
|
||||||
|
|
||||||
if isinstance(item, list):
|
if isinstance(item, list):
|
||||||
for i, subitem in enumerate(item):
|
for index, subitem in enumerate(item):
|
||||||
item[i] = resolve_env_variables(subitem)
|
item[index] = resolve_env_variables(subitem)
|
||||||
|
|
||||||
if isinstance(item, dict):
|
if isinstance(item, dict):
|
||||||
for key, value in item.items():
|
for key, value in item.items():
|
||||||
item[key] = resolve_env_variables(value)
|
item[key] = resolve_env_variables(value)
|
||||||
|
|
||||||
return item
|
return item
|
||||||
|
|
|
@ -3,7 +3,7 @@ import io
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from ruamel import yaml
|
import ruamel.yaml
|
||||||
|
|
||||||
from borgmatic.config import load, normalize
|
from borgmatic.config import load, normalize
|
||||||
|
|
||||||
|
@ -17,10 +17,23 @@ def insert_newline_before_comment(config, field_name):
|
||||||
field and its comments.
|
field and its comments.
|
||||||
'''
|
'''
|
||||||
config.ca.items[field_name][1].insert(
|
config.ca.items[field_name][1].insert(
|
||||||
0, yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None)
|
0, ruamel.yaml.tokens.CommentToken('\n', ruamel.yaml.error.CommentMark(0), None)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_properties(schema):
|
||||||
|
'''
|
||||||
|
Given a schema dict, return its properties. But if it's got sub-schemas with multiple different
|
||||||
|
potential properties, returned their merged properties instead.
|
||||||
|
'''
|
||||||
|
if 'oneOf' in schema:
|
||||||
|
return dict(
|
||||||
|
collections.ChainMap(*[sub_schema['properties'] for sub_schema in schema['oneOf']])
|
||||||
|
)
|
||||||
|
|
||||||
|
return schema['properties']
|
||||||
|
|
||||||
|
|
||||||
def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
|
def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
|
||||||
'''
|
'''
|
||||||
Given a loaded configuration schema, generate and return sample config for it. Include comments
|
Given a loaded configuration schema, generate and return sample config for it. Include comments
|
||||||
|
@ -32,15 +45,15 @@ def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
|
||||||
return example
|
return example
|
||||||
|
|
||||||
if schema_type == 'array':
|
if schema_type == 'array':
|
||||||
config = yaml.comments.CommentedSeq(
|
config = ruamel.yaml.comments.CommentedSeq(
|
||||||
[schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
|
[schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
|
||||||
)
|
)
|
||||||
add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
|
add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
|
||||||
elif schema_type == 'object':
|
elif schema_type == 'object':
|
||||||
config = yaml.comments.CommentedMap(
|
config = ruamel.yaml.comments.CommentedMap(
|
||||||
[
|
[
|
||||||
(field_name, schema_to_sample_configuration(sub_schema, level + 1))
|
(field_name, schema_to_sample_configuration(sub_schema, level + 1))
|
||||||
for field_name, sub_schema in schema['properties'].items()
|
for field_name, sub_schema in get_properties(schema).items()
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
|
indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
|
||||||
|
@ -101,7 +114,7 @@ def render_configuration(config):
|
||||||
'''
|
'''
|
||||||
Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
|
Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
|
||||||
'''
|
'''
|
||||||
dumper = yaml.YAML()
|
dumper = ruamel.yaml.YAML(typ='rt')
|
||||||
dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
|
dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
|
||||||
rendered = io.StringIO()
|
rendered = io.StringIO()
|
||||||
dumper.dump(config, rendered)
|
dumper.dump(config, rendered)
|
||||||
|
@ -151,7 +164,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
|
||||||
return
|
return
|
||||||
|
|
||||||
for field_name in config[0].keys():
|
for field_name in config[0].keys():
|
||||||
field_schema = schema['items']['properties'].get(field_name, {})
|
field_schema = get_properties(schema['items']).get(field_name, {})
|
||||||
description = field_schema.get('description')
|
description = field_schema.get('description')
|
||||||
|
|
||||||
# No description to use? Skip it.
|
# No description to use? Skip it.
|
||||||
|
@ -178,7 +191,7 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa
|
||||||
if skip_first and index == 0:
|
if skip_first and index == 0:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
field_schema = schema['properties'].get(field_name, {})
|
field_schema = get_properties(schema).get(field_name, {})
|
||||||
description = field_schema.get('description', '').strip()
|
description = field_schema.get('description', '').strip()
|
||||||
|
|
||||||
# If this is an optional key, add an indicator to the comment flagging it to be commented
|
# If this is an optional key, add an indicator to the comment flagging it to be commented
|
||||||
|
@ -225,8 +238,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
|
||||||
favoring values from the source when there are collisions.
|
favoring values from the source when there are collisions.
|
||||||
|
|
||||||
The purpose of this is to upgrade configuration files from old versions of borgmatic by adding
|
The purpose of this is to upgrade configuration files from old versions of borgmatic by adding
|
||||||
new
|
new configuration keys and comments.
|
||||||
configuration keys and comments.
|
|
||||||
'''
|
'''
|
||||||
if not source_config:
|
if not source_config:
|
||||||
return destination_config
|
return destination_config
|
||||||
|
@ -236,7 +248,9 @@ def merge_source_configuration_into_destination(destination_config, source_confi
|
||||||
for field_name, source_value in source_config.items():
|
for field_name, source_value in source_config.items():
|
||||||
# Since this key/value is from the source configuration, leave it uncommented and remove any
|
# Since this key/value is from the source configuration, leave it uncommented and remove any
|
||||||
# sentinel that would cause it to get commented out.
|
# sentinel that would cause it to get commented out.
|
||||||
remove_commented_out_sentinel(destination_config, field_name)
|
remove_commented_out_sentinel(
|
||||||
|
ruamel.yaml.comments.CommentedMap(destination_config), field_name
|
||||||
|
)
|
||||||
|
|
||||||
# This is a mapping. Recurse for this key/value.
|
# This is a mapping. Recurse for this key/value.
|
||||||
if isinstance(source_value, collections.abc.Mapping):
|
if isinstance(source_value, collections.abc.Mapping):
|
||||||
|
@ -248,7 +262,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
|
||||||
# This is a sequence. Recurse for each item in it.
|
# This is a sequence. Recurse for each item in it.
|
||||||
if isinstance(source_value, collections.abc.Sequence) and not isinstance(source_value, str):
|
if isinstance(source_value, collections.abc.Sequence) and not isinstance(source_value, str):
|
||||||
destination_value = destination_config[field_name]
|
destination_value = destination_config[field_name]
|
||||||
destination_config[field_name] = yaml.comments.CommentedSeq(
|
destination_config[field_name] = ruamel.yaml.comments.CommentedSeq(
|
||||||
[
|
[
|
||||||
merge_source_configuration_into_destination(
|
merge_source_configuration_into_destination(
|
||||||
destination_value[index] if index < len(destination_value) else None,
|
destination_value[index] if index < len(destination_value) else None,
|
||||||
|
@ -275,7 +289,7 @@ def generate_sample_configuration(
|
||||||
schema. If a source filename is provided, merge the parsed contents of that configuration into
|
schema. If a source filename is provided, merge the parsed contents of that configuration into
|
||||||
the generated configuration.
|
the generated configuration.
|
||||||
'''
|
'''
|
||||||
schema = yaml.round_trip_load(open(schema_filename))
|
schema = ruamel.yaml.YAML(typ='safe').load(open(schema_filename))
|
||||||
source_config = None
|
source_config = None
|
||||||
|
|
||||||
if source_filename:
|
if source_filename:
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
import functools
|
import functools
|
||||||
import itertools
|
import itertools
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
import operator
|
import operator
|
||||||
import os
|
import os
|
||||||
|
@ -10,18 +9,18 @@ import ruamel.yaml
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def probe_and_include_file(filename, include_directories):
|
def probe_and_include_file(filename, include_directories, config_paths):
|
||||||
'''
|
'''
|
||||||
Given a filename to include and a list of include directories to search for matching files,
|
Given a filename to include, a list of include directories to search for matching files, and a
|
||||||
probe for the file, load it, and return the loaded configuration as a data structure of nested
|
set of configuration paths, probe for the file, load it, and return the loaded configuration as
|
||||||
dicts, lists, etc.
|
a data structure of nested dicts, lists, etc. Add the filename to the given configuration paths.
|
||||||
|
|
||||||
Raise FileNotFoundError if the included file was not found.
|
Raise FileNotFoundError if the included file was not found.
|
||||||
'''
|
'''
|
||||||
expanded_filename = os.path.expanduser(filename)
|
expanded_filename = os.path.expanduser(filename)
|
||||||
|
|
||||||
if os.path.isabs(expanded_filename):
|
if os.path.isabs(expanded_filename):
|
||||||
return load_configuration(expanded_filename)
|
return load_configuration(expanded_filename, config_paths)
|
||||||
|
|
||||||
candidate_filenames = {
|
candidate_filenames = {
|
||||||
os.path.join(directory, expanded_filename) for directory in include_directories
|
os.path.join(directory, expanded_filename) for directory in include_directories
|
||||||
|
@ -29,32 +28,33 @@ def probe_and_include_file(filename, include_directories):
|
||||||
|
|
||||||
for candidate_filename in candidate_filenames:
|
for candidate_filename in candidate_filenames:
|
||||||
if os.path.exists(candidate_filename):
|
if os.path.exists(candidate_filename):
|
||||||
return load_configuration(candidate_filename)
|
return load_configuration(candidate_filename, config_paths)
|
||||||
|
|
||||||
raise FileNotFoundError(
|
raise FileNotFoundError(
|
||||||
f'Could not find include {filename} at {" or ".join(candidate_filenames)}'
|
f'Could not find include {filename} at {" or ".join(candidate_filenames)}'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def include_configuration(loader, filename_node, include_directory):
|
def include_configuration(loader, filename_node, include_directory, config_paths):
|
||||||
'''
|
'''
|
||||||
Given a ruamel.yaml.loader.Loader, a ruamel.yaml.nodes.ScalarNode containing the included
|
Given a ruamel.yaml.loader.Loader, a ruamel.yaml.nodes.ScalarNode containing the included
|
||||||
filename (or a list containing multiple such filenames), and an include directory path to search
|
filename (or a list containing multiple such filenames), an include directory path to search for
|
||||||
for matching files, load the given YAML filenames (ignoring the given loader so we can use our
|
matching files, and a set of configuration paths, load the given YAML filenames (ignoring the
|
||||||
own) and return their contents as data structure of nested dicts, lists, etc. If the given
|
given loader so we can use our own) and return their contents as data structure of nested dicts,
|
||||||
|
lists, etc. Add the names of included files to the given configuration paths. If the given
|
||||||
filename node's value is a scalar string, then the return value will be a single value. But if
|
filename node's value is a scalar string, then the return value will be a single value. But if
|
||||||
the given node value is a list, then the return value will be a list of values, one per loaded
|
the given node value is a list, then the return value will be a list of values, one per loaded
|
||||||
configuration file.
|
configuration file.
|
||||||
|
|
||||||
If a filename is relative, probe for it within 1. the current working directory and 2. the given
|
If a filename is relative, probe for it within: 1. the current working directory and 2. the
|
||||||
include directory.
|
given include directory.
|
||||||
|
|
||||||
Raise FileNotFoundError if an included file was not found.
|
Raise FileNotFoundError if an included file was not found.
|
||||||
'''
|
'''
|
||||||
include_directories = [os.getcwd(), os.path.abspath(include_directory)]
|
include_directories = [os.getcwd(), os.path.abspath(include_directory)]
|
||||||
|
|
||||||
if isinstance(filename_node.value, str):
|
if isinstance(filename_node.value, str):
|
||||||
return probe_and_include_file(filename_node.value, include_directories)
|
return probe_and_include_file(filename_node.value, include_directories, config_paths)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
isinstance(filename_node.value, list)
|
isinstance(filename_node.value, list)
|
||||||
|
@ -64,7 +64,7 @@ def include_configuration(loader, filename_node, include_directory):
|
||||||
# Reversing the values ensures the correct ordering if these includes are subsequently
|
# Reversing the values ensures the correct ordering if these includes are subsequently
|
||||||
# merged together.
|
# merged together.
|
||||||
return [
|
return [
|
||||||
probe_and_include_file(node.value, include_directories)
|
probe_and_include_file(node.value, include_directories, config_paths)
|
||||||
for node in reversed(filename_node.value)
|
for node in reversed(filename_node.value)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -110,11 +110,17 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
|
||||||
separate YAML configuration files. Example syntax: `option: !include common.yaml`
|
separate YAML configuration files. Example syntax: `option: !include common.yaml`
|
||||||
'''
|
'''
|
||||||
|
|
||||||
def __init__(self, preserve_quotes=None, loader=None, include_directory=None):
|
def __init__(
|
||||||
|
self, preserve_quotes=None, loader=None, include_directory=None, config_paths=None
|
||||||
|
):
|
||||||
super(Include_constructor, self).__init__(preserve_quotes, loader)
|
super(Include_constructor, self).__init__(preserve_quotes, loader)
|
||||||
self.add_constructor(
|
self.add_constructor(
|
||||||
'!include',
|
'!include',
|
||||||
functools.partial(include_configuration, include_directory=include_directory),
|
functools.partial(
|
||||||
|
include_configuration,
|
||||||
|
include_directory=include_directory,
|
||||||
|
config_paths=config_paths,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# These are catch-all error handlers for tags that don't get applied and removed by
|
# These are catch-all error handlers for tags that don't get applied and removed by
|
||||||
|
@ -156,46 +162,36 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
|
||||||
node.value = deep_merge_nodes(node.value)
|
node.value = deep_merge_nodes(node.value)
|
||||||
|
|
||||||
|
|
||||||
def load_configuration(filename):
|
def load_configuration(filename, config_paths=None):
|
||||||
'''
|
'''
|
||||||
Load the given configuration file and return its contents as a data structure of nested dicts
|
Load the given configuration file and return its contents as a data structure of nested dicts
|
||||||
and lists. Also, replace any "{constant}" strings with the value of the "constant" key in the
|
and lists. Add the filename to the given configuration paths set, and also add any included
|
||||||
"constants" option of the configuration file.
|
configuration filenames.
|
||||||
|
|
||||||
Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
|
Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
|
||||||
if there are too many recursive includes.
|
if there are too many recursive includes.
|
||||||
'''
|
'''
|
||||||
|
if config_paths is None:
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
# Use an embedded derived class for the include constructor so as to capture the filename
|
# Use an embedded derived class for the include constructor so as to capture the include
|
||||||
# value. (functools.partial doesn't work for this use case because yaml.Constructor has to be
|
# directory and configuration paths values. (functools.partial doesn't work for this use case
|
||||||
# an actual class.)
|
# because yaml.Constructor has to be an actual class.)
|
||||||
class Include_constructor_with_include_directory(Include_constructor):
|
class Include_constructor_with_extras(Include_constructor):
|
||||||
def __init__(self, preserve_quotes=None, loader=None):
|
def __init__(self, preserve_quotes=None, loader=None):
|
||||||
super(Include_constructor_with_include_directory, self).__init__(
|
super(Include_constructor_with_extras, self).__init__(
|
||||||
preserve_quotes, loader, include_directory=os.path.dirname(filename)
|
preserve_quotes,
|
||||||
|
loader,
|
||||||
|
include_directory=os.path.dirname(filename),
|
||||||
|
config_paths=config_paths,
|
||||||
)
|
)
|
||||||
|
|
||||||
yaml = ruamel.yaml.YAML(typ='safe')
|
yaml = ruamel.yaml.YAML(typ='safe')
|
||||||
yaml.Constructor = Include_constructor_with_include_directory
|
yaml.Constructor = Include_constructor_with_extras
|
||||||
|
config_paths.add(filename)
|
||||||
|
|
||||||
with open(filename) as file:
|
with open(filename) as file:
|
||||||
file_contents = file.read()
|
return yaml.load(file.read())
|
||||||
config = yaml.load(file_contents)
|
|
||||||
|
|
||||||
try:
|
|
||||||
has_constants = bool(config and 'constants' in config)
|
|
||||||
except TypeError:
|
|
||||||
has_constants = False
|
|
||||||
|
|
||||||
if has_constants:
|
|
||||||
for key, value in config['constants'].items():
|
|
||||||
value = json.dumps(value)
|
|
||||||
file_contents = file_contents.replace(f'{{{key}}}', value.strip('"'))
|
|
||||||
|
|
||||||
config = yaml.load(file_contents)
|
|
||||||
del config['constants']
|
|
||||||
|
|
||||||
return config
|
|
||||||
|
|
||||||
|
|
||||||
def filter_omitted_nodes(nodes, values):
|
def filter_omitted_nodes(nodes, values):
|
||||||
|
|
|
@ -39,7 +39,7 @@ def normalize_sections(config_filename, config):
|
||||||
for section_name in ('location', 'storage', 'retention', 'consistency', 'output', 'hooks'):
|
for section_name in ('location', 'storage', 'retention', 'consistency', 'output', 'hooks'):
|
||||||
section_config = config.get(section_name)
|
section_config = config.get(section_name)
|
||||||
|
|
||||||
if section_config:
|
if section_config is not None:
|
||||||
any_section_upgraded = True
|
any_section_upgraded = True
|
||||||
del config[section_name]
|
del config[section_name]
|
||||||
config.update(section_config)
|
config.update(section_config)
|
||||||
|
@ -50,7 +50,7 @@ def normalize_sections(config_filename, config):
|
||||||
dict(
|
dict(
|
||||||
levelno=logging.WARNING,
|
levelno=logging.WARNING,
|
||||||
levelname='WARNING',
|
levelname='WARNING',
|
||||||
msg=f'{config_filename}: Configuration sections (like location: and storage:) are deprecated and support will be removed from a future release. To prepare for this, move your options out of sections to the global scope.',
|
msg=f'{config_filename}: Configuration sections (like location:, storage:, retention:, consistency:, and hooks:) are deprecated and support will be removed from a future release. To prepare for this, move your options out of sections to the global scope.',
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
@ -90,7 +90,7 @@ def normalize(config_filename, config):
|
||||||
dict(
|
dict(
|
||||||
levelno=logging.WARNING,
|
levelno=logging.WARNING,
|
||||||
levelname='WARNING',
|
levelname='WARNING',
|
||||||
msg=f'{config_filename}: The healthchecks hook now expects a mapping value. String values for this option are deprecated and support will be removed from a future release.',
|
msg=f'{config_filename}: The healthchecks hook now expects a key/value pair with "ping_url" as a key. String values for this option are deprecated and support will be removed from a future release.',
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
@ -192,7 +192,7 @@ def normalize(config_filename, config):
|
||||||
# Upgrade remote repositories to ssh:// syntax, required in Borg 2.
|
# Upgrade remote repositories to ssh:// syntax, required in Borg 2.
|
||||||
repositories = config.get('repositories')
|
repositories = config.get('repositories')
|
||||||
if repositories:
|
if repositories:
|
||||||
if isinstance(repositories[0], str):
|
if any(isinstance(repository, str) for repository in repositories):
|
||||||
logs.append(
|
logs.append(
|
||||||
logging.makeLogRecord(
|
logging.makeLogRecord(
|
||||||
dict(
|
dict(
|
||||||
|
@ -202,7 +202,10 @@ def normalize(config_filename, config):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
config['repositories'] = [{'path': repository} for repository in repositories]
|
config['repositories'] = [
|
||||||
|
{'path': repository} if isinstance(repository, str) else repository
|
||||||
|
for repository in repositories
|
||||||
|
]
|
||||||
repositories = config['repositories']
|
repositories = config['repositories']
|
||||||
|
|
||||||
config['repositories'] = []
|
config['repositories'] = []
|
||||||
|
|
|
@ -13,6 +13,11 @@ def set_values(config, keys, value):
|
||||||
|
|
||||||
first_key = keys[0]
|
first_key = keys[0]
|
||||||
if len(keys) == 1:
|
if len(keys) == 1:
|
||||||
|
if isinstance(config, list):
|
||||||
|
raise ValueError(
|
||||||
|
'When overriding a list option, the value must use list syntax (e.g., "[foo, bar]" or "[{key: value}]" as appropriate)'
|
||||||
|
)
|
||||||
|
|
||||||
config[first_key] = value
|
config[first_key] = value
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -22,13 +27,19 @@ def set_values(config, keys, value):
|
||||||
set_values(config[first_key], keys[1:], value)
|
set_values(config[first_key], keys[1:], value)
|
||||||
|
|
||||||
|
|
||||||
def convert_value_type(value):
|
def convert_value_type(value, option_type):
|
||||||
'''
|
'''
|
||||||
Given a string value, determine its logical type (string, boolean, integer, etc.), and return it
|
Given a string value and its schema type as a string, determine its logical type (string,
|
||||||
converted to that type.
|
boolean, integer, etc.), and return it converted to that type.
|
||||||
|
|
||||||
|
If the option type is a string, leave the value as a string so that special characters in it
|
||||||
|
don't get interpreted as YAML during conversion.
|
||||||
|
|
||||||
Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML.
|
Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML.
|
||||||
'''
|
'''
|
||||||
|
if option_type == 'string':
|
||||||
|
return value
|
||||||
|
|
||||||
return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
|
return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
|
||||||
|
|
||||||
|
|
||||||
|
@ -46,11 +57,32 @@ def strip_section_names(parsed_override_key):
|
||||||
return parsed_override_key
|
return parsed_override_key
|
||||||
|
|
||||||
|
|
||||||
def parse_overrides(raw_overrides):
|
def type_for_option(schema, option_keys):
|
||||||
'''
|
'''
|
||||||
Given a sequence of configuration file override strings in the form of "option.suboption=value",
|
Given a configuration schema and a sequence of keys identifying an option, e.g.
|
||||||
parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For
|
('extra_borg_options', 'init'), return the schema type of that option as a string.
|
||||||
instance, given the following raw overrides:
|
|
||||||
|
Return None if the option or its type cannot be found in the schema.
|
||||||
|
'''
|
||||||
|
option_schema = schema
|
||||||
|
|
||||||
|
for key in option_keys:
|
||||||
|
try:
|
||||||
|
option_schema = option_schema['properties'][key]
|
||||||
|
except KeyError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
return option_schema['type']
|
||||||
|
except KeyError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_overrides(raw_overrides, schema):
|
||||||
|
'''
|
||||||
|
Given a sequence of configuration file override strings in the form of "option.suboption=value"
|
||||||
|
and a configuration schema dict, parse and return a sequence of tuples (keys, values), where
|
||||||
|
keys is a sequence of strings. For instance, given the following raw overrides:
|
||||||
|
|
||||||
['my_option.suboption=value1', 'other_option=value2']
|
['my_option.suboption=value1', 'other_option=value2']
|
||||||
|
|
||||||
|
@ -71,10 +103,13 @@ def parse_overrides(raw_overrides):
|
||||||
for raw_override in raw_overrides:
|
for raw_override in raw_overrides:
|
||||||
try:
|
try:
|
||||||
raw_keys, value = raw_override.split('=', 1)
|
raw_keys, value = raw_override.split('=', 1)
|
||||||
|
keys = tuple(raw_keys.split('.'))
|
||||||
|
option_type = type_for_option(schema, keys)
|
||||||
|
|
||||||
parsed_overrides.append(
|
parsed_overrides.append(
|
||||||
(
|
(
|
||||||
strip_section_names(tuple(raw_keys.split('.'))),
|
keys,
|
||||||
convert_value_type(value),
|
convert_value_type(value, option_type),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
|
@ -87,12 +122,18 @@ def parse_overrides(raw_overrides):
|
||||||
return tuple(parsed_overrides)
|
return tuple(parsed_overrides)
|
||||||
|
|
||||||
|
|
||||||
def apply_overrides(config, raw_overrides):
|
def apply_overrides(config, schema, raw_overrides):
|
||||||
'''
|
'''
|
||||||
Given a configuration dict and a sequence of configuration file override strings in the form of
|
Given a configuration dict, a corresponding configuration schema dict, and a sequence of
|
||||||
"option.suboption=value", parse each override and set it the configuration dict.
|
configuration file override strings in the form of "option.suboption=value", parse each override
|
||||||
|
and set it into the configuration dict.
|
||||||
|
|
||||||
|
Set the overrides into the configuration both with and without deprecated section names (if
|
||||||
|
used), so that the overrides work regardless of whether the configuration is also using
|
||||||
|
deprecated section names.
|
||||||
'''
|
'''
|
||||||
overrides = parse_overrides(raw_overrides)
|
overrides = parse_overrides(raw_overrides, schema)
|
||||||
|
|
||||||
for keys, value in overrides:
|
for keys, value in overrides:
|
||||||
set_values(config, keys, value)
|
set_values(config, keys, value)
|
||||||
|
set_values(config, strip_section_names(keys), value)
|
||||||
|
|
|
@ -6,14 +6,15 @@ properties:
|
||||||
constants:
|
constants:
|
||||||
type: object
|
type: object
|
||||||
description: |
|
description: |
|
||||||
Constants to use in the configuration file. All occurrences of the
|
Constants to use in the configuration file. Within option values,
|
||||||
constant name within curly braces will be replaced with the value.
|
all occurrences of the constant name in curly braces will be
|
||||||
For example, if you have a constant named "hostname" with the value
|
replaced with the constant value. For example, if you have a
|
||||||
"myhostname", then the string "{hostname}" will be replaced with
|
constant named "app_name" with the value "myapp", then the string
|
||||||
"myhostname" in the configuration file.
|
"{app_name}" will be replaced with "myapp" in the configuration
|
||||||
|
file.
|
||||||
example:
|
example:
|
||||||
hostname: myhostname
|
app_name: myapp
|
||||||
prefix: myprefix
|
user: myuser
|
||||||
source_directories:
|
source_directories:
|
||||||
type: array
|
type: array
|
||||||
items:
|
items:
|
||||||
|
@ -216,7 +217,7 @@ properties:
|
||||||
Store configuration files used to create a backup in the backup
|
Store configuration files used to create a backup in the backup
|
||||||
itself. Defaults to true. Changing this to false prevents "borgmatic
|
itself. Defaults to true. Changing this to false prevents "borgmatic
|
||||||
bootstrap" from extracting configuration files from the backup.
|
bootstrap" from extracting configuration files from the backup.
|
||||||
example: true
|
example: false
|
||||||
source_directories_must_exist:
|
source_directories_must_exist:
|
||||||
type: boolean
|
type: boolean
|
||||||
description: |
|
description: |
|
||||||
|
@ -268,7 +269,8 @@ properties:
|
||||||
compression:
|
compression:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
Type of compression to use when creating archives. See
|
Type of compression to use when creating archives. (Compression
|
||||||
|
level can be added separated with a comma, like "zstd,7".) See
|
||||||
http://borgbackup.readthedocs.io/en/stable/usage/create.html for
|
http://borgbackup.readthedocs.io/en/stable/usage/create.html for
|
||||||
details. Defaults to "lz4".
|
details. Defaults to "lz4".
|
||||||
example: lz4
|
example: lz4
|
||||||
|
@ -287,14 +289,17 @@ properties:
|
||||||
retry_wait:
|
retry_wait:
|
||||||
type: integer
|
type: integer
|
||||||
description: |
|
description: |
|
||||||
Wait time between retries (in seconds) to allow transient issues to
|
Wait time between retries (in seconds) to allow transient issues
|
||||||
pass. Increases after each retry as a form of backoff. Defaults to 0
|
to pass. Increases after each retry by that same wait time as a
|
||||||
(no wait).
|
form of backoff. Defaults to 0 (no wait).
|
||||||
example: 10
|
example: 10
|
||||||
temporary_directory:
|
temporary_directory:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
Directory where temporary files are stored. Defaults to $TMPDIR.
|
Directory where temporary Borg files are stored. Defaults to
|
||||||
|
$TMPDIR. See "Resource Usage" at
|
||||||
|
https://borgbackup.readthedocs.io/en/stable/usage/general.html for
|
||||||
|
details.
|
||||||
example: /path/to/tmpdir
|
example: /path/to/tmpdir
|
||||||
ssh_command:
|
ssh_command:
|
||||||
type: string
|
type: string
|
||||||
|
@ -337,6 +342,37 @@ properties:
|
||||||
Path for Borg encryption key files. Defaults to
|
Path for Borg encryption key files. Defaults to
|
||||||
$borg_base_directory/.config/borg/keys
|
$borg_base_directory/.config/borg/keys
|
||||||
example: /path/to/base/config/keys
|
example: /path/to/base/config/keys
|
||||||
|
borg_exit_codes:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: object
|
||||||
|
required: ['code', 'treat_as']
|
||||||
|
additionalProperties: false
|
||||||
|
properties:
|
||||||
|
code:
|
||||||
|
type: integer
|
||||||
|
not: {enum: [0]}
|
||||||
|
description: |
|
||||||
|
The exit code for an existing Borg warning or error.
|
||||||
|
example: 100
|
||||||
|
treat_as:
|
||||||
|
type: string
|
||||||
|
enum: ['error', 'warning']
|
||||||
|
description: |
|
||||||
|
Whether to consider the exit code as an error or as a
|
||||||
|
warning in borgmatic.
|
||||||
|
example: error
|
||||||
|
description: |
|
||||||
|
A list of Borg exit codes that should be elevated to errors or
|
||||||
|
squashed to warnings as indicated. By default, Borg error exit codes
|
||||||
|
(2 to 99) are treated as errors while warning exit codes (1 and
|
||||||
|
100+) are treated as warnings. Exit codes other than 1 and 2 are
|
||||||
|
only present in Borg 1.4.0+.
|
||||||
|
example:
|
||||||
|
- code: 13
|
||||||
|
treat_as: warning
|
||||||
|
- code: 100
|
||||||
|
treat_as: error
|
||||||
umask:
|
umask:
|
||||||
type: integer
|
type: integer
|
||||||
description: |
|
description: |
|
||||||
|
@ -423,7 +459,9 @@ properties:
|
||||||
command-line invocation.
|
command-line invocation.
|
||||||
keep_within:
|
keep_within:
|
||||||
type: string
|
type: string
|
||||||
description: Keep all archives within this time interval.
|
description: |
|
||||||
|
Keep all archives within this time interval. See "skip_actions" for
|
||||||
|
disabling pruning altogether.
|
||||||
example: 3H
|
example: 3H
|
||||||
keep_secondly:
|
keep_secondly:
|
||||||
type: integer
|
type: integer
|
||||||
|
@ -466,37 +504,120 @@ properties:
|
||||||
type: array
|
type: array
|
||||||
items:
|
items:
|
||||||
type: object
|
type: object
|
||||||
required: ['name']
|
oneOf:
|
||||||
additionalProperties: false
|
- required: [name]
|
||||||
properties:
|
additionalProperties: false
|
||||||
name:
|
properties:
|
||||||
type: string
|
name:
|
||||||
enum:
|
type: string
|
||||||
- repository
|
enum:
|
||||||
- archives
|
- repository
|
||||||
- data
|
- archives
|
||||||
- extract
|
- data
|
||||||
- disabled
|
- extract
|
||||||
description: |
|
- disabled
|
||||||
Name of consistency check to run: "repository",
|
description: |
|
||||||
"archives", "data", and/or "extract". Set to "disabled"
|
Name of consistency check to run: "repository",
|
||||||
to disable all consistency checks. "repository" checks
|
"archives", "data", "spot", and/or "extract".
|
||||||
the consistency of the repository, "archives" checks all
|
"repository" checks the consistency of the
|
||||||
of the archives, "data" verifies the integrity of the
|
repository, "archives" checks all of the
|
||||||
data within the archives, and "extract" does an
|
archives, "data" verifies the integrity of the
|
||||||
extraction dry-run of the most recent archive. Note that
|
data within the archives, "spot" checks that
|
||||||
"data" implies "archives".
|
some percentage of source files are found in the
|
||||||
example: repository
|
most recent archive (with identical contents),
|
||||||
frequency:
|
and "extract" does an extraction dry-run of the
|
||||||
type: string
|
most recent archive. Note that "data" implies
|
||||||
description: |
|
"archives". See "skip_actions" for disabling
|
||||||
How frequently to run this type of consistency check (as
|
checks altogether.
|
||||||
a best effort). The value is a number followed by a unit
|
example: spot
|
||||||
of time. E.g., "2 weeks" to run this consistency check
|
frequency:
|
||||||
no more than every two weeks for a given repository or
|
type: string
|
||||||
"1 month" to run it no more than monthly. Defaults to
|
description: |
|
||||||
"always": running this check every time checks are run.
|
How frequently to run this type of consistency
|
||||||
example: 2 weeks
|
check (as a best effort). The value is a number
|
||||||
|
followed by a unit of time. E.g., "2 weeks" to
|
||||||
|
run this consistency check no more than every
|
||||||
|
two weeks for a given repository or "1 month" to
|
||||||
|
run it no more than monthly. Defaults to
|
||||||
|
"always": running this check every time checks
|
||||||
|
are run.
|
||||||
|
example: 2 weeks
|
||||||
|
- required:
|
||||||
|
- name
|
||||||
|
- count_tolerance_percentage
|
||||||
|
- data_sample_percentage
|
||||||
|
- data_tolerance_percentage
|
||||||
|
additionalProperties: false
|
||||||
|
properties:
|
||||||
|
name:
|
||||||
|
type: string
|
||||||
|
enum:
|
||||||
|
- spot
|
||||||
|
description: |
|
||||||
|
Name of consistency check to run: "repository",
|
||||||
|
"archives", "data", "spot", and/or "extract".
|
||||||
|
"repository" checks the consistency of the
|
||||||
|
repository, "archives" checks all of the
|
||||||
|
archives, "data" verifies the integrity of the
|
||||||
|
data within the archives, "spot" checks that
|
||||||
|
some percentage of source files are found in the
|
||||||
|
most recent archive (with identical contents),
|
||||||
|
and "extract" does an extraction dry-run of the
|
||||||
|
most recent archive. Note that "data" implies
|
||||||
|
"archives". See "skip_actions" for disabling
|
||||||
|
checks altogether.
|
||||||
|
example: repository
|
||||||
|
frequency:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
How frequently to run this type of consistency
|
||||||
|
check (as a best effort). The value is a number
|
||||||
|
followed by a unit of time. E.g., "2 weeks" to
|
||||||
|
run this consistency check no more than every
|
||||||
|
two weeks for a given repository or "1 month" to
|
||||||
|
run it no more than monthly. Defaults to
|
||||||
|
"always": running this check every time checks
|
||||||
|
are run.
|
||||||
|
example: 2 weeks
|
||||||
|
count_tolerance_percentage:
|
||||||
|
type: number
|
||||||
|
description: |
|
||||||
|
The percentage delta between the source
|
||||||
|
directories file count and the most recent backup
|
||||||
|
archive file count that is allowed before the
|
||||||
|
entire consistency check fails. This can catch
|
||||||
|
problems like incorrect excludes, inadvertent
|
||||||
|
deletes, etc. Only applies to the "spot" check.
|
||||||
|
example: 10
|
||||||
|
data_sample_percentage:
|
||||||
|
type: number
|
||||||
|
description: |
|
||||||
|
The percentage of total files in the source
|
||||||
|
directories to randomly sample and compare to
|
||||||
|
their corresponding files in the most recent
|
||||||
|
backup archive. Only applies to the "spot" check.
|
||||||
|
example: 1
|
||||||
|
data_tolerance_percentage:
|
||||||
|
type: number
|
||||||
|
description: |
|
||||||
|
The percentage of total files in the source
|
||||||
|
directories that can fail a spot check comparison
|
||||||
|
without failing the entire consistency check. This
|
||||||
|
can catch problems like source files that have
|
||||||
|
been bulk-changed by malware, backups that have
|
||||||
|
been tampered with, etc. The value must be lower
|
||||||
|
than or equal to the "contents_sample_percentage".
|
||||||
|
Only applies to the "spot" check.
|
||||||
|
example: 0.5
|
||||||
|
xxh64sum_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to use instead of "xxh64sum" to hash
|
||||||
|
source files, usually found in an OS package named
|
||||||
|
"xxhash". Do not substitute with a different hash
|
||||||
|
type (SHA, MD5, etc.) or the check will never
|
||||||
|
succeed. Only applies to the "spot" check.
|
||||||
|
example: /usr/local/bin/xxh64sum
|
||||||
description: |
|
description: |
|
||||||
List of one or more consistency checks to run on a periodic basis
|
List of one or more consistency checks to run on a periodic basis
|
||||||
(if "frequency" is set) or every time borgmatic runs checks (if
|
(if "frequency" is set) or every time borgmatic runs checks (if
|
||||||
|
@ -525,6 +646,38 @@ properties:
|
||||||
Apply color to console output. Can be overridden with --no-color
|
Apply color to console output. Can be overridden with --no-color
|
||||||
command-line flag. Defaults to true.
|
command-line flag. Defaults to true.
|
||||||
example: false
|
example: false
|
||||||
|
skip_actions:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
enum:
|
||||||
|
- rcreate
|
||||||
|
- transfer
|
||||||
|
- prune
|
||||||
|
- compact
|
||||||
|
- create
|
||||||
|
- check
|
||||||
|
- extract
|
||||||
|
- config
|
||||||
|
- export-tar
|
||||||
|
- mount
|
||||||
|
- umount
|
||||||
|
- restore
|
||||||
|
- rlist
|
||||||
|
- list
|
||||||
|
- rinfo
|
||||||
|
- info
|
||||||
|
- break-lock
|
||||||
|
- key
|
||||||
|
- borg
|
||||||
|
description: |
|
||||||
|
List of one or more actions to skip running for this configuration
|
||||||
|
file, even if specified on the command-line (explicitly or
|
||||||
|
implicitly). This is handy for append-only configurations where you
|
||||||
|
never want to run "compact" or checkless configuration where you
|
||||||
|
want to skip "check". Defaults to not skipping any actions.
|
||||||
|
example:
|
||||||
|
- compact
|
||||||
before_actions:
|
before_actions:
|
||||||
type: array
|
type: array
|
||||||
items:
|
items:
|
||||||
|
@ -902,6 +1055,20 @@ properties:
|
||||||
a password will only work if MariaDB is configured to
|
a password will only work if MariaDB is configured to
|
||||||
trust the configured username without a password.
|
trust the configured username without a password.
|
||||||
example: trustsome1
|
example: trustsome1
|
||||||
|
mariadb_dump_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to use instead of "mariadb-dump". This can be
|
||||||
|
used to run a specific mariadb_dump version (e.g., one
|
||||||
|
inside a running container). Defaults to "mariadb-dump".
|
||||||
|
example: docker exec mariadb_container mariadb-dump
|
||||||
|
mariadb_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to run instead of "mariadb". This can be used to
|
||||||
|
run a specific mariadb version (e.g., one inside a
|
||||||
|
running container). Defaults to "mariadb".
|
||||||
|
example: docker exec mariadb_container mariadb
|
||||||
restore_password:
|
restore_password:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
|
@ -1018,6 +1185,20 @@ properties:
|
||||||
Password with which to connect to the restore database.
|
Password with which to connect to the restore database.
|
||||||
Defaults to the "password" option.
|
Defaults to the "password" option.
|
||||||
example: trustsome1
|
example: trustsome1
|
||||||
|
mysql_dump_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to use instead of "mysqldump". This can be used
|
||||||
|
to run a specific mysql_dump version (e.g., one inside a
|
||||||
|
running container). Defaults to "mysqldump".
|
||||||
|
example: docker exec mysql_container mysqldump
|
||||||
|
mysql_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to run instead of "mysql". This can be used to
|
||||||
|
run a specific mysql version (e.g., one inside a running
|
||||||
|
container). Defaults to "mysql".
|
||||||
|
example: docker exec mysql_container mysql
|
||||||
format:
|
format:
|
||||||
type: string
|
type: string
|
||||||
enum: ['sql']
|
enum: ['sql']
|
||||||
|
@ -1221,6 +1402,12 @@ properties:
|
||||||
description: |
|
description: |
|
||||||
The password used for authentication.
|
The password used for authentication.
|
||||||
example: fakepassword
|
example: fakepassword
|
||||||
|
access_token:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
An ntfy access token to authenticate with instead of
|
||||||
|
username/password.
|
||||||
|
example: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2
|
||||||
start:
|
start:
|
||||||
type: object
|
type: object
|
||||||
properties:
|
properties:
|
||||||
|
@ -1306,6 +1493,129 @@ properties:
|
||||||
example:
|
example:
|
||||||
- start
|
- start
|
||||||
- finish
|
- finish
|
||||||
|
apprise:
|
||||||
|
type: object
|
||||||
|
required: ['services']
|
||||||
|
additionalProperties: false
|
||||||
|
properties:
|
||||||
|
services:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- url
|
||||||
|
- label
|
||||||
|
properties:
|
||||||
|
url:
|
||||||
|
type: string
|
||||||
|
example: "gotify://hostname/token"
|
||||||
|
label:
|
||||||
|
type: string
|
||||||
|
example: gotify
|
||||||
|
description: |
|
||||||
|
A list of Apprise services to publish to with URLs and
|
||||||
|
labels. The labels are used for logging. A full list of
|
||||||
|
services and their configuration can be found at
|
||||||
|
https://github.com/caronc/apprise/wiki.
|
||||||
|
example:
|
||||||
|
- url: "kodi://user@hostname"
|
||||||
|
label: kodi
|
||||||
|
- url: "line://Token@User"
|
||||||
|
label: line
|
||||||
|
send_logs:
|
||||||
|
type: boolean
|
||||||
|
description: |
|
||||||
|
Send borgmatic logs to Apprise services as part the
|
||||||
|
"finish", "fail", and "log" states. Defaults to true.
|
||||||
|
example: false
|
||||||
|
logs_size_limit:
|
||||||
|
type: integer
|
||||||
|
description: |
|
||||||
|
Number of bytes of borgmatic logs to send to Apprise
|
||||||
|
services. Set to 0 to send all logs and disable this
|
||||||
|
truncation. Defaults to 1500.
|
||||||
|
example: 100000
|
||||||
|
start:
|
||||||
|
type: object
|
||||||
|
required: ['body']
|
||||||
|
properties:
|
||||||
|
title:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Specify the message title. If left unspecified, no
|
||||||
|
title is sent.
|
||||||
|
example: Ping!
|
||||||
|
body:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Specify the message body.
|
||||||
|
example: Starting backup process.
|
||||||
|
finish:
|
||||||
|
type: object
|
||||||
|
required: ['body']
|
||||||
|
properties:
|
||||||
|
title:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Specify the message title. If left unspecified, no
|
||||||
|
title is sent.
|
||||||
|
example: Ping!
|
||||||
|
body:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Specify the message body.
|
||||||
|
example: Backups successfully made.
|
||||||
|
fail:
|
||||||
|
type: object
|
||||||
|
required: ['body']
|
||||||
|
properties:
|
||||||
|
title:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Specify the message title. If left unspecified, no
|
||||||
|
title is sent.
|
||||||
|
example: Ping!
|
||||||
|
body:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Specify the message body.
|
||||||
|
example: Your backups have failed.
|
||||||
|
log:
|
||||||
|
type: object
|
||||||
|
required: ['body']
|
||||||
|
properties:
|
||||||
|
title:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Specify the message title. If left unspecified, no
|
||||||
|
title is sent.
|
||||||
|
example: Ping!
|
||||||
|
body:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Specify the message body.
|
||||||
|
example: Here is some info about your backups.
|
||||||
|
states:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
enum:
|
||||||
|
- start
|
||||||
|
- finish
|
||||||
|
- fail
|
||||||
|
- log
|
||||||
|
uniqueItems: true
|
||||||
|
description: |
|
||||||
|
List of one or more monitoring states to ping for:
|
||||||
|
"start", "finish", "fail", and/or "log". Defaults to
|
||||||
|
pinging for failure only. For each selected state,
|
||||||
|
corresponding configuration for the message title and body
|
||||||
|
should be given. If any is left unspecified, a generic
|
||||||
|
message is emitted instead.
|
||||||
|
example:
|
||||||
|
- start
|
||||||
|
- finish
|
||||||
|
|
||||||
healthchecks:
|
healthchecks:
|
||||||
type: object
|
type: object
|
||||||
required: ['ping_url']
|
required: ['ping_url']
|
||||||
|
@ -1353,6 +1663,14 @@ properties:
|
||||||
states.
|
states.
|
||||||
example:
|
example:
|
||||||
- finish
|
- finish
|
||||||
|
create_slug:
|
||||||
|
type: boolean
|
||||||
|
description: |
|
||||||
|
Create the check if it does not exist. Only works with
|
||||||
|
the slug URL scheme (https://hc-ping.com/<ping-key>/<slug>
|
||||||
|
as opposed to https://hc-ping.com/<uuid>).
|
||||||
|
Defaults to false.
|
||||||
|
example: true
|
||||||
description: |
|
description: |
|
||||||
Configuration for a monitoring integration with Healthchecks. Create
|
Configuration for a monitoring integration with Healthchecks. Create
|
||||||
an account at https://healthchecks.io (or self-host Healthchecks) if
|
an account at https://healthchecks.io (or self-host Healthchecks) if
|
||||||
|
@ -1400,6 +1718,36 @@ properties:
|
||||||
ends, or errors.
|
ends, or errors.
|
||||||
example: https://cronhub.io/ping/1f5e3410-254c-5587
|
example: https://cronhub.io/ping/1f5e3410-254c-5587
|
||||||
description: |
|
description: |
|
||||||
Configuration for a monitoring integration with Crunhub. Create an
|
Configuration for a monitoring integration with Cronhub. Create an
|
||||||
account at https://cronhub.io if you'd like to use this service. See
|
account at https://cronhub.io if you'd like to use this service. See
|
||||||
borgmatic monitoring documentation for details.
|
borgmatic monitoring documentation for details.
|
||||||
|
loki:
|
||||||
|
type: object
|
||||||
|
required: ['url', 'labels']
|
||||||
|
additionalProperties: false
|
||||||
|
properties:
|
||||||
|
url:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Grafana loki log URL to notify when a backup begins,
|
||||||
|
ends, or fails.
|
||||||
|
example: "http://localhost:3100/loki/api/v1/push"
|
||||||
|
labels:
|
||||||
|
type: object
|
||||||
|
additionalProperties:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Allows setting custom labels for the logging stream. At
|
||||||
|
least one label is required. "__hostname" gets replaced by
|
||||||
|
the machine hostname automatically. "__config" gets replaced
|
||||||
|
by just the name of the configuration file. "__config_path"
|
||||||
|
gets replaced by the full path of the configuration file.
|
||||||
|
example:
|
||||||
|
app: "borgmatic"
|
||||||
|
config: "__config"
|
||||||
|
hostname: "__hostname"
|
||||||
|
description: |
|
||||||
|
Configuration for a monitoring integration with Grafana loki. You
|
||||||
|
can send the logs to a self-hosted instance or create an account at
|
||||||
|
https://grafana.com/auth/sign-up/create-user. See borgmatic
|
||||||
|
monitoring documentation for details.
|
||||||
|
|
|
@ -4,7 +4,7 @@ import jsonschema
|
||||||
import ruamel.yaml
|
import ruamel.yaml
|
||||||
|
|
||||||
import borgmatic.config
|
import borgmatic.config
|
||||||
from borgmatic.config import environment, load, normalize, override
|
from borgmatic.config import constants, environment, load, normalize, override
|
||||||
|
|
||||||
|
|
||||||
def schema_filename():
|
def schema_filename():
|
||||||
|
@ -97,23 +97,28 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
|
||||||
'checks': ['repository', 'archives'],
|
'checks': ['repository', 'archives'],
|
||||||
}
|
}
|
||||||
|
|
||||||
Also return a sequence of logging.LogRecord instances containing any warnings about the
|
Also return a set of loaded configuration paths and a sequence of logging.LogRecord instances
|
||||||
configuration.
|
containing any warnings about the configuration.
|
||||||
|
|
||||||
Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
|
Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
|
||||||
have permissions to read the file, or Validation_error if the config does not match the schema.
|
have permissions to read the file, or Validation_error if the config does not match the schema.
|
||||||
'''
|
'''
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
config = load.load_configuration(config_filename)
|
config = load.load_configuration(config_filename, config_paths)
|
||||||
schema = load.load_configuration(schema_filename)
|
schema = load.load_configuration(schema_filename)
|
||||||
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
|
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
|
||||||
raise Validation_error(config_filename, (str(error),))
|
raise Validation_error(config_filename, (str(error),))
|
||||||
|
|
||||||
override.apply_overrides(config, overrides)
|
override.apply_overrides(config, schema, overrides)
|
||||||
logs = normalize.normalize(config_filename, config)
|
constants.apply_constants(config, config.get('constants') if config else {})
|
||||||
|
|
||||||
if resolve_env:
|
if resolve_env:
|
||||||
environment.resolve_env_variables(config)
|
environment.resolve_env_variables(config)
|
||||||
|
|
||||||
|
logs = normalize.normalize(config_filename, config)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
validator = jsonschema.Draft7Validator(schema)
|
validator = jsonschema.Draft7Validator(schema)
|
||||||
except AttributeError: # pragma: no cover
|
except AttributeError: # pragma: no cover
|
||||||
|
@ -127,7 +132,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
|
||||||
|
|
||||||
apply_logical_validation(config_filename, config)
|
apply_logical_validation(config_filename, config)
|
||||||
|
|
||||||
return config, logs
|
return config, config_paths, logs
|
||||||
|
|
||||||
|
|
||||||
def normalize_repository_path(repository):
|
def normalize_repository_path(repository):
|
||||||
|
@ -162,11 +167,10 @@ def repositories_match(first, second):
|
||||||
def guard_configuration_contains_repository(repository, configurations):
|
def guard_configuration_contains_repository(repository, configurations):
|
||||||
'''
|
'''
|
||||||
Given a repository path and a dict mapping from config filename to corresponding parsed config
|
Given a repository path and a dict mapping from config filename to corresponding parsed config
|
||||||
dict, ensure that the repository is declared exactly once in all of the configurations. If no
|
dict, ensure that the repository is declared at least once in all of the configurations. If no
|
||||||
repository is given, skip this check.
|
repository is given, skip this check.
|
||||||
|
|
||||||
Raise ValueError if the repository is not found in a configuration, or is declared multiple
|
Raise ValueError if the repository is not found in any configurations.
|
||||||
times.
|
|
||||||
'''
|
'''
|
||||||
if not repository:
|
if not repository:
|
||||||
return
|
return
|
||||||
|
@ -181,9 +185,7 @@ def guard_configuration_contains_repository(repository, configurations):
|
||||||
)
|
)
|
||||||
|
|
||||||
if count == 0:
|
if count == 0:
|
||||||
raise ValueError(f'Repository {repository} not found in configuration files')
|
raise ValueError(f'Repository "{repository}" not found in configuration files')
|
||||||
if count > 1:
|
|
||||||
raise ValueError(f'Repository {repository} found in multiple configuration files')
|
|
||||||
|
|
||||||
|
|
||||||
def guard_single_repository_selected(repository, configurations):
|
def guard_single_repository_selected(repository, configurations):
|
||||||
|
|
|
@ -1,29 +1,70 @@
|
||||||
import collections
|
import collections
|
||||||
|
import enum
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import select
|
import select
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import textwrap
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
ERROR_OUTPUT_MAX_LINE_COUNT = 25
|
ERROR_OUTPUT_MAX_LINE_COUNT = 25
|
||||||
BORG_ERROR_EXIT_CODE = 2
|
BORG_ERROR_EXIT_CODE_START = 2
|
||||||
|
BORG_ERROR_EXIT_CODE_END = 99
|
||||||
|
|
||||||
|
|
||||||
def exit_code_indicates_error(command, exit_code, borg_local_path=None):
|
class Exit_status(enum.Enum):
|
||||||
|
STILL_RUNNING = 1
|
||||||
|
SUCCESS = 2
|
||||||
|
WARNING = 3
|
||||||
|
ERROR = 4
|
||||||
|
|
||||||
|
|
||||||
|
def interpret_exit_code(command, exit_code, borg_local_path=None, borg_exit_codes=None):
|
||||||
'''
|
'''
|
||||||
Return True if the given exit code from running a command corresponds to an error. If a Borg
|
Return an Exit_status value (e.g. SUCCESS, ERROR, or WARNING) based on interpreting the given
|
||||||
local path is given and matches the process' command, then treat exit code 1 as a warning
|
exit code. If a Borg local path is given and matches the process' command, then interpret the
|
||||||
instead of an error.
|
exit code based on Borg's documented exit code semantics. And if Borg exit codes are given as a
|
||||||
|
sequence of exit code configuration dicts, then take those configured preferences into account.
|
||||||
'''
|
'''
|
||||||
if exit_code is None:
|
if exit_code is None:
|
||||||
return False
|
return Exit_status.STILL_RUNNING
|
||||||
|
if exit_code == 0:
|
||||||
|
return Exit_status.SUCCESS
|
||||||
|
|
||||||
if borg_local_path and command[0] == borg_local_path:
|
if borg_local_path and command[0] == borg_local_path:
|
||||||
return bool(exit_code < 0 or exit_code >= BORG_ERROR_EXIT_CODE)
|
# First try looking for the exit code in the borg_exit_codes configuration.
|
||||||
|
for entry in borg_exit_codes or ():
|
||||||
|
if entry.get('code') == exit_code:
|
||||||
|
treat_as = entry.get('treat_as')
|
||||||
|
|
||||||
return bool(exit_code != 0)
|
if treat_as == 'error':
|
||||||
|
logger.error(
|
||||||
|
f'Treating exit code {exit_code} as an error, as per configuration'
|
||||||
|
)
|
||||||
|
return Exit_status.ERROR
|
||||||
|
elif treat_as == 'warning':
|
||||||
|
logger.warning(
|
||||||
|
f'Treating exit code {exit_code} as a warning, as per configuration'
|
||||||
|
)
|
||||||
|
return Exit_status.WARNING
|
||||||
|
|
||||||
|
# If the exit code doesn't have explicit configuration, then fall back to the default Borg
|
||||||
|
# behavior.
|
||||||
|
return (
|
||||||
|
Exit_status.ERROR
|
||||||
|
if (
|
||||||
|
exit_code < 0
|
||||||
|
or (
|
||||||
|
exit_code >= BORG_ERROR_EXIT_CODE_START
|
||||||
|
and exit_code <= BORG_ERROR_EXIT_CODE_END
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else Exit_status.WARNING
|
||||||
|
)
|
||||||
|
|
||||||
|
return Exit_status.ERROR
|
||||||
|
|
||||||
|
|
||||||
def command_for_process(process):
|
def command_for_process(process):
|
||||||
|
@ -60,7 +101,7 @@ def append_last_lines(last_lines, captured_output, line, output_log_level):
|
||||||
logger.log(output_log_level, line)
|
logger.log(output_log_level, line)
|
||||||
|
|
||||||
|
|
||||||
def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
|
def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, borg_exit_codes):
|
||||||
'''
|
'''
|
||||||
Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
|
Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
|
||||||
process with the requested log level. Additionally, raise a CalledProcessError if a process
|
process with the requested log level. Additionally, raise a CalledProcessError if a process
|
||||||
|
@ -68,7 +109,8 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
|
||||||
path).
|
path).
|
||||||
|
|
||||||
If output log level is None, then instead of logging, capture output for each process and return
|
If output log level is None, then instead of logging, capture output for each process and return
|
||||||
it as a dict from the process to its output.
|
it as a dict from the process to its output. Use the given Borg local path and exit code
|
||||||
|
configuration to decide what's an error and what's a warning.
|
||||||
|
|
||||||
For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
|
For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
|
||||||
stdouts are given to exclude, then for any matching processes, log from their stderr instead.
|
stdouts are given to exclude, then for any matching processes, log from their stderr instead.
|
||||||
|
@ -132,10 +174,13 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
|
||||||
|
|
||||||
if exit_code is None:
|
if exit_code is None:
|
||||||
still_running = True
|
still_running = True
|
||||||
|
command = process.args.split(' ') if isinstance(process.args, str) else process.args
|
||||||
|
continue
|
||||||
|
|
||||||
command = process.args.split(' ') if isinstance(process.args, str) else process.args
|
command = process.args.split(' ') if isinstance(process.args, str) else process.args
|
||||||
# If any process errors, then raise accordingly.
|
exit_status = interpret_exit_code(command, exit_code, borg_local_path, borg_exit_codes)
|
||||||
if exit_code_indicates_error(command, exit_code, borg_local_path):
|
|
||||||
|
if exit_status in (Exit_status.ERROR, Exit_status.WARNING):
|
||||||
# If an error occurs, include its output in the raised exception so that we don't
|
# If an error occurs, include its output in the raised exception so that we don't
|
||||||
# inadvertently hide error output.
|
# inadvertently hide error output.
|
||||||
output_buffer = output_buffer_for_process(process, exclude_stdouts)
|
output_buffer = output_buffer_for_process(process, exclude_stdouts)
|
||||||
|
@ -161,9 +206,13 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
|
||||||
other_process.stdout.read(0)
|
other_process.stdout.read(0)
|
||||||
other_process.kill()
|
other_process.kill()
|
||||||
|
|
||||||
raise subprocess.CalledProcessError(
|
if exit_status == Exit_status.ERROR:
|
||||||
exit_code, command_for_process(process), '\n'.join(last_lines)
|
raise subprocess.CalledProcessError(
|
||||||
)
|
exit_code, command_for_process(process), '\n'.join(last_lines)
|
||||||
|
)
|
||||||
|
|
||||||
|
still_running = False
|
||||||
|
break
|
||||||
|
|
||||||
if captured_outputs:
|
if captured_outputs:
|
||||||
return {
|
return {
|
||||||
|
@ -171,19 +220,47 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def log_command(full_command, input_file=None, output_file=None):
|
SECRET_COMMAND_FLAG_NAMES = {'--password'}
|
||||||
|
|
||||||
|
|
||||||
|
def mask_command_secrets(full_command):
|
||||||
|
'''
|
||||||
|
Given a command as a sequence, mask secret values for flags like "--password" in preparation for
|
||||||
|
logging.
|
||||||
|
'''
|
||||||
|
masked_command = []
|
||||||
|
previous_piece = None
|
||||||
|
|
||||||
|
for piece in full_command:
|
||||||
|
masked_command.append('***' if previous_piece in SECRET_COMMAND_FLAG_NAMES else piece)
|
||||||
|
previous_piece = piece
|
||||||
|
|
||||||
|
return tuple(masked_command)
|
||||||
|
|
||||||
|
|
||||||
|
MAX_LOGGED_COMMAND_LENGTH = 1000
|
||||||
|
|
||||||
|
|
||||||
|
def log_command(full_command, input_file=None, output_file=None, environment=None):
|
||||||
'''
|
'''
|
||||||
Log the given command (a sequence of command/argument strings), along with its input/output file
|
Log the given command (a sequence of command/argument strings), along with its input/output file
|
||||||
paths.
|
paths and extra environment variables (with omitted values in case they contain passwords).
|
||||||
'''
|
'''
|
||||||
logger.debug(
|
logger.debug(
|
||||||
' '.join(full_command)
|
textwrap.shorten(
|
||||||
|
' '.join(
|
||||||
|
tuple(f'{key}=***' for key in (environment or {}).keys())
|
||||||
|
+ mask_command_secrets(full_command)
|
||||||
|
),
|
||||||
|
width=MAX_LOGGED_COMMAND_LENGTH,
|
||||||
|
placeholder=' ...',
|
||||||
|
)
|
||||||
+ (f" < {getattr(input_file, 'name', '')}" if input_file else '')
|
+ (f" < {getattr(input_file, 'name', '')}" if input_file else '')
|
||||||
+ (f" > {getattr(output_file, 'name', '')}" if output_file else '')
|
+ (f" > {getattr(output_file, 'name', '')}" if output_file else '')
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# An sentinel passed as an output file to execute_command() to indicate that the command's output
|
# A sentinel passed as an output file to execute_command() to indicate that the command's output
|
||||||
# should be allowed to flow through to stdout without being captured for logging. Useful for
|
# should be allowed to flow through to stdout without being captured for logging. Useful for
|
||||||
# commands with interactive prompts or those that mess directly with the console.
|
# commands with interactive prompts or those that mess directly with the console.
|
||||||
DO_NOT_CAPTURE = object()
|
DO_NOT_CAPTURE = object()
|
||||||
|
@ -198,6 +275,7 @@ def execute_command(
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
working_directory=None,
|
working_directory=None,
|
||||||
borg_local_path=None,
|
borg_local_path=None,
|
||||||
|
borg_exit_codes=None,
|
||||||
run_to_completion=True,
|
run_to_completion=True,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
|
@ -208,12 +286,13 @@ def execute_command(
|
||||||
augment the current environment, and pass the result into the command. If a working directory is
|
augment the current environment, and pass the result into the command. If a working directory is
|
||||||
given, use that as the present working directory when running the command. If a Borg local path
|
given, use that as the present working directory when running the command. If a Borg local path
|
||||||
is given, and the command matches it (regardless of arguments), treat exit code 1 as a warning
|
is given, and the command matches it (regardless of arguments), treat exit code 1 as a warning
|
||||||
instead of an error. If run to completion is False, then return the process for the command
|
instead of an error. But if Borg exit codes are given as a sequence of exit code configuration
|
||||||
without executing it to completion.
|
dicts, then use that configuration to decide what's an error and what's a warning. If run to
|
||||||
|
completion is False, then return the process for the command without executing it to completion.
|
||||||
|
|
||||||
Raise subprocesses.CalledProcessError if an error occurs while running the command.
|
Raise subprocesses.CalledProcessError if an error occurs while running the command.
|
||||||
'''
|
'''
|
||||||
log_command(full_command, input_file, output_file)
|
log_command(full_command, input_file, output_file, extra_environment)
|
||||||
environment = {**os.environ, **extra_environment} if extra_environment else None
|
environment = {**os.environ, **extra_environment} if extra_environment else None
|
||||||
do_not_capture = bool(output_file is DO_NOT_CAPTURE)
|
do_not_capture = bool(output_file is DO_NOT_CAPTURE)
|
||||||
command = ' '.join(full_command) if shell else full_command
|
command = ' '.join(full_command) if shell else full_command
|
||||||
|
@ -231,7 +310,11 @@ def execute_command(
|
||||||
return process
|
return process
|
||||||
|
|
||||||
log_outputs(
|
log_outputs(
|
||||||
(process,), (input_file, output_file), output_log_level, borg_local_path=borg_local_path
|
(process,),
|
||||||
|
(input_file, output_file),
|
||||||
|
output_log_level,
|
||||||
|
borg_local_path,
|
||||||
|
borg_exit_codes,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -242,6 +325,7 @@ def execute_command_and_capture_output(
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
working_directory=None,
|
working_directory=None,
|
||||||
borg_local_path=None,
|
borg_local_path=None,
|
||||||
|
borg_exit_codes=None,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Execute the given command (a sequence of command/argument strings), capturing and returning its
|
Execute the given command (a sequence of command/argument strings), capturing and returning its
|
||||||
|
@ -250,11 +334,13 @@ def execute_command_and_capture_output(
|
||||||
given, then use it to augment the current environment, and pass the result into the command. If
|
given, then use it to augment the current environment, and pass the result into the command. If
|
||||||
a working directory is given, use that as the present working directory when running the
|
a working directory is given, use that as the present working directory when running the
|
||||||
command. If a Borg local path is given, and the command matches it (regardless of arguments),
|
command. If a Borg local path is given, and the command matches it (regardless of arguments),
|
||||||
treat exit code 1 as a warning instead of an error.
|
treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a
|
||||||
|
sequence of exit code configuration dicts, then use that configuration to decide what's an error
|
||||||
|
and what's a warning.
|
||||||
|
|
||||||
Raise subprocesses.CalledProcessError if an error occurs while running the command.
|
Raise subprocesses.CalledProcessError if an error occurs while running the command.
|
||||||
'''
|
'''
|
||||||
log_command(full_command)
|
log_command(full_command, environment=extra_environment)
|
||||||
environment = {**os.environ, **extra_environment} if extra_environment else None
|
environment = {**os.environ, **extra_environment} if extra_environment else None
|
||||||
command = ' '.join(full_command) if shell else full_command
|
command = ' '.join(full_command) if shell else full_command
|
||||||
|
|
||||||
|
@ -267,7 +353,10 @@ def execute_command_and_capture_output(
|
||||||
cwd=working_directory,
|
cwd=working_directory,
|
||||||
)
|
)
|
||||||
except subprocess.CalledProcessError as error:
|
except subprocess.CalledProcessError as error:
|
||||||
if exit_code_indicates_error(command, error.returncode, borg_local_path):
|
if (
|
||||||
|
interpret_exit_code(command, error.returncode, borg_local_path, borg_exit_codes)
|
||||||
|
== Exit_status.ERROR
|
||||||
|
):
|
||||||
raise
|
raise
|
||||||
output = error.output
|
output = error.output
|
||||||
|
|
||||||
|
@ -284,6 +373,7 @@ def execute_command_with_processes(
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
working_directory=None,
|
working_directory=None,
|
||||||
borg_local_path=None,
|
borg_local_path=None,
|
||||||
|
borg_exit_codes=None,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Execute the given command (a sequence of command/argument strings) and log its output at the
|
Execute the given command (a sequence of command/argument strings) and log its output at the
|
||||||
|
@ -298,12 +388,14 @@ def execute_command_with_processes(
|
||||||
use it to augment the current environment, and pass the result into the command. If a working
|
use it to augment the current environment, and pass the result into the command. If a working
|
||||||
directory is given, use that as the present working directory when running the command. If a
|
directory is given, use that as the present working directory when running the command. If a
|
||||||
Borg local path is given, then for any matching command or process (regardless of arguments),
|
Borg local path is given, then for any matching command or process (regardless of arguments),
|
||||||
treat exit code 1 as a warning instead of an error.
|
treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a
|
||||||
|
sequence of exit code configuration dicts, then use that configuration to decide what's an error
|
||||||
|
and what's a warning.
|
||||||
|
|
||||||
Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
|
Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
|
||||||
upstream process.
|
upstream process.
|
||||||
'''
|
'''
|
||||||
log_command(full_command, input_file, output_file)
|
log_command(full_command, input_file, output_file, extra_environment)
|
||||||
environment = {**os.environ, **extra_environment} if extra_environment else None
|
environment = {**os.environ, **extra_environment} if extra_environment else None
|
||||||
do_not_capture = bool(output_file is DO_NOT_CAPTURE)
|
do_not_capture = bool(output_file is DO_NOT_CAPTURE)
|
||||||
command = ' '.join(full_command) if shell else full_command
|
command = ' '.join(full_command) if shell else full_command
|
||||||
|
@ -313,9 +405,9 @@ def execute_command_with_processes(
|
||||||
command,
|
command,
|
||||||
stdin=input_file,
|
stdin=input_file,
|
||||||
stdout=None if do_not_capture else (output_file or subprocess.PIPE),
|
stdout=None if do_not_capture else (output_file or subprocess.PIPE),
|
||||||
stderr=None
|
stderr=(
|
||||||
if do_not_capture
|
None if do_not_capture else (subprocess.PIPE if output_file else subprocess.STDOUT)
|
||||||
else (subprocess.PIPE if output_file else subprocess.STDOUT),
|
),
|
||||||
shell=shell,
|
shell=shell,
|
||||||
env=environment,
|
env=environment,
|
||||||
cwd=working_directory,
|
cwd=working_directory,
|
||||||
|
@ -333,7 +425,8 @@ def execute_command_with_processes(
|
||||||
tuple(processes) + (command_process,),
|
tuple(processes) + (command_process,),
|
||||||
(input_file, output_file),
|
(input_file, output_file),
|
||||||
output_log_level,
|
output_log_level,
|
||||||
borg_local_path=borg_local_path,
|
borg_local_path,
|
||||||
|
borg_exit_codes,
|
||||||
)
|
)
|
||||||
|
|
||||||
if output_log_level is None:
|
if output_log_level is None:
|
||||||
|
|
|
@ -0,0 +1,109 @@
|
||||||
|
import logging
|
||||||
|
import operator
|
||||||
|
|
||||||
|
import borgmatic.hooks.logs
|
||||||
|
import borgmatic.hooks.monitor
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_LOGS_SIZE_LIMIT_BYTES = 100000
|
||||||
|
HANDLER_IDENTIFIER = 'apprise'
|
||||||
|
|
||||||
|
|
||||||
|
def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
|
||||||
|
'''
|
||||||
|
Add a handler to the root logger that stores in memory the most recent logs emitted. That way,
|
||||||
|
we can send them all to an Apprise notification service upon a finish or failure state. But skip
|
||||||
|
this if the "send_logs" option is false.
|
||||||
|
'''
|
||||||
|
if hook_config.get('send_logs') is False:
|
||||||
|
return
|
||||||
|
|
||||||
|
logs_size_limit = max(
|
||||||
|
hook_config.get('logs_size_limit', DEFAULT_LOGS_SIZE_LIMIT_BYTES)
|
||||||
|
- len(borgmatic.hooks.logs.PAYLOAD_TRUNCATION_INDICATOR),
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
|
||||||
|
borgmatic.hooks.logs.add_handler(
|
||||||
|
borgmatic.hooks.logs.Forgetful_buffering_handler(
|
||||||
|
HANDLER_IDENTIFIER, logs_size_limit, monitoring_log_level
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
|
||||||
|
'''
|
||||||
|
Ping the configured Apprise service URLs. Use the given configuration filename in any log
|
||||||
|
entries. If this is a dry run, then don't actually ping anything.
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
import apprise
|
||||||
|
from apprise import NotifyFormat, NotifyType
|
||||||
|
except ImportError: # pragma: no cover
|
||||||
|
logger.warning('Unable to import Apprise in monitoring hook')
|
||||||
|
return
|
||||||
|
|
||||||
|
state_to_notify_type = {
|
||||||
|
'start': NotifyType.INFO,
|
||||||
|
'finish': NotifyType.SUCCESS,
|
||||||
|
'fail': NotifyType.FAILURE,
|
||||||
|
'log': NotifyType.INFO,
|
||||||
|
}
|
||||||
|
|
||||||
|
run_states = hook_config.get('states', ['fail'])
|
||||||
|
|
||||||
|
if state.name.lower() not in run_states:
|
||||||
|
return
|
||||||
|
|
||||||
|
state_config = hook_config.get(
|
||||||
|
state.name.lower(),
|
||||||
|
{
|
||||||
|
'title': f'A borgmatic {state.name} event happened',
|
||||||
|
'body': f'A borgmatic {state.name} event happened',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
if not hook_config.get('services'):
|
||||||
|
logger.info(f'{config_filename}: No Apprise services to ping')
|
||||||
|
return
|
||||||
|
|
||||||
|
dry_run_string = ' (dry run; not actually pinging)' if dry_run else ''
|
||||||
|
labels_string = ', '.join(map(operator.itemgetter('label'), hook_config.get('services')))
|
||||||
|
logger.info(f'{config_filename}: Pinging Apprise services: {labels_string}{dry_run_string}')
|
||||||
|
|
||||||
|
apprise_object = apprise.Apprise()
|
||||||
|
apprise_object.add(list(map(operator.itemgetter('url'), hook_config.get('services'))))
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
return
|
||||||
|
|
||||||
|
body = state_config.get('body')
|
||||||
|
|
||||||
|
if state in (
|
||||||
|
borgmatic.hooks.monitor.State.FINISH,
|
||||||
|
borgmatic.hooks.monitor.State.FAIL,
|
||||||
|
borgmatic.hooks.monitor.State.LOG,
|
||||||
|
):
|
||||||
|
formatted_logs = borgmatic.hooks.logs.format_buffered_logs_for_payload(HANDLER_IDENTIFIER)
|
||||||
|
if formatted_logs:
|
||||||
|
body += f'\n\n{formatted_logs}'
|
||||||
|
|
||||||
|
result = apprise_object.notify(
|
||||||
|
title=state_config.get('title', ''),
|
||||||
|
body=body,
|
||||||
|
body_format=NotifyFormat.TEXT,
|
||||||
|
notify_type=state_to_notify_type[state.name.lower()],
|
||||||
|
)
|
||||||
|
|
||||||
|
if result is False:
|
||||||
|
logger.warning(f'{config_filename}: Error sending some Apprise notifications')
|
||||||
|
|
||||||
|
|
||||||
|
def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
|
||||||
|
'''
|
||||||
|
Remove the monitor handler that was added to the root logger. This prevents the handler from
|
||||||
|
getting reused by other instances of this monitor.
|
||||||
|
'''
|
||||||
|
borgmatic.hooks.logs.remove_handler(HANDLER_IDENTIFIER)
|
|
@ -1,6 +1,7 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import shlex
|
||||||
|
|
||||||
from borgmatic import execute
|
from borgmatic import execute
|
||||||
|
|
||||||
|
@ -16,7 +17,7 @@ def interpolate_context(config_filename, hook_description, command, context):
|
||||||
names/values, interpolate the values by "{name}" into the command and return the result.
|
names/values, interpolate the values by "{name}" into the command and return the result.
|
||||||
'''
|
'''
|
||||||
for name, value in context.items():
|
for name, value in context.items():
|
||||||
command = command.replace(f'{{{name}}}', str(value))
|
command = command.replace(f'{{{name}}}', shlex.quote(str(value)))
|
||||||
|
|
||||||
for unsupported_variable in re.findall(r'{\w+}', command):
|
for unsupported_variable in re.findall(r'{\w+}', command):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
|
@ -67,9 +68,9 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
execute.execute_command(
|
execute.execute_command(
|
||||||
[command],
|
[command],
|
||||||
output_log_level=logging.ERROR
|
output_log_level=(
|
||||||
if description == 'on-error'
|
logging.ERROR if description == 'on-error' else logging.WARNING
|
||||||
else logging.WARNING,
|
),
|
||||||
shell=True,
|
shell=True,
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from borgmatic.hooks import (
|
from borgmatic.hooks import (
|
||||||
|
apprise,
|
||||||
cronhub,
|
cronhub,
|
||||||
cronitor,
|
cronitor,
|
||||||
healthchecks,
|
healthchecks,
|
||||||
|
loki,
|
||||||
mariadb,
|
mariadb,
|
||||||
mongodb,
|
mongodb,
|
||||||
mysql,
|
mysql,
|
||||||
|
@ -16,6 +18,7 @@ from borgmatic.hooks import (
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
HOOK_NAME_TO_MODULE = {
|
HOOK_NAME_TO_MODULE = {
|
||||||
|
'apprise': apprise,
|
||||||
'cronhub': cronhub,
|
'cronhub': cronhub,
|
||||||
'cronitor': cronitor,
|
'cronitor': cronitor,
|
||||||
'healthchecks': healthchecks,
|
'healthchecks': healthchecks,
|
||||||
|
@ -26,6 +29,7 @@ HOOK_NAME_TO_MODULE = {
|
||||||
'pagerduty': pagerduty,
|
'pagerduty': pagerduty,
|
||||||
'postgresql_databases': postgresql,
|
'postgresql_databases': postgresql,
|
||||||
'sqlite_databases': sqlite,
|
'sqlite_databases': sqlite,
|
||||||
|
'loki': loki,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
DATABASE_HOOK_NAMES = (
|
DATA_SOURCE_HOOK_NAMES = (
|
||||||
'mariadb_databases',
|
'mariadb_databases',
|
||||||
'mysql_databases',
|
'mysql_databases',
|
||||||
'mongodb_databases',
|
'mongodb_databases',
|
||||||
|
@ -15,26 +15,26 @@ DATABASE_HOOK_NAMES = (
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_path(borgmatic_source_directory, database_hook_name):
|
def make_data_source_dump_path(borgmatic_source_directory, data_source_hook_name):
|
||||||
'''
|
'''
|
||||||
Given a borgmatic source directory (or None) and a database hook name, construct a database dump
|
Given a borgmatic source directory (or None) and a data source hook name, construct a data
|
||||||
path.
|
source dump path.
|
||||||
'''
|
'''
|
||||||
if not borgmatic_source_directory:
|
if not borgmatic_source_directory:
|
||||||
borgmatic_source_directory = DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
borgmatic_source_directory = DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||||
|
|
||||||
return os.path.join(borgmatic_source_directory, database_hook_name)
|
return os.path.join(borgmatic_source_directory, data_source_hook_name)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_filename(dump_path, name, hostname=None):
|
def make_data_source_dump_filename(dump_path, name, hostname=None):
|
||||||
'''
|
'''
|
||||||
Based on the given dump directory path, database name, and hostname, return a filename to use
|
Based on the given dump directory path, data source name, and hostname, return a filename to use
|
||||||
for the database dump. The hostname defaults to localhost.
|
for the data source dump. The hostname defaults to localhost.
|
||||||
|
|
||||||
Raise ValueError if the database name is invalid.
|
Raise ValueError if the data source name is invalid.
|
||||||
'''
|
'''
|
||||||
if os.path.sep in name:
|
if os.path.sep in name:
|
||||||
raise ValueError(f'Invalid database name {name}')
|
raise ValueError(f'Invalid data source name {name}')
|
||||||
|
|
||||||
return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
|
return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
|
||||||
|
|
||||||
|
@ -54,14 +54,14 @@ def create_named_pipe_for_dump(dump_path):
|
||||||
os.mkfifo(dump_path, mode=0o600)
|
os.mkfifo(dump_path, mode=0o600)
|
||||||
|
|
||||||
|
|
||||||
def remove_database_dumps(dump_path, database_type_name, log_prefix, dry_run):
|
def remove_data_source_dumps(dump_path, data_source_type_name, log_prefix, dry_run):
|
||||||
'''
|
'''
|
||||||
Remove all database dumps in the given dump directory path (including the directory itself). If
|
Remove all data source dumps in the given dump directory path (including the directory itself).
|
||||||
this is a dry run, then don't actually remove anything.
|
If this is a dry run, then don't actually remove anything.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
|
||||||
|
|
||||||
logger.debug(f'{log_prefix}: Removing {database_type_name} database dumps{dry_run_label}')
|
logger.debug(f'{log_prefix}: Removing {data_source_type_name} data source dumps{dry_run_label}')
|
||||||
|
|
||||||
expanded_path = os.path.expanduser(dump_path)
|
expanded_path = os.path.expanduser(dump_path)
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
import borgmatic.hooks.logs
|
||||||
from borgmatic.hooks import monitor
|
from borgmatic.hooks import monitor
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -13,61 +15,8 @@ MONITOR_STATE_TO_HEALTHCHECKS = {
|
||||||
monitor.State.LOG: 'log',
|
monitor.State.LOG: 'log',
|
||||||
}
|
}
|
||||||
|
|
||||||
PAYLOAD_TRUNCATION_INDICATOR = '...\n'
|
DEFAULT_PING_BODY_LIMIT_BYTES = 1500
|
||||||
DEFAULT_PING_BODY_LIMIT_BYTES = 100000
|
HANDLER_IDENTIFIER = 'healthchecks'
|
||||||
|
|
||||||
|
|
||||||
class Forgetful_buffering_handler(logging.Handler):
|
|
||||||
'''
|
|
||||||
A buffering log handler that stores log messages in memory, and throws away messages (oldest
|
|
||||||
first) once a particular capacity in bytes is reached. But if the given byte capacity is zero,
|
|
||||||
don't throw away any messages.
|
|
||||||
'''
|
|
||||||
|
|
||||||
def __init__(self, byte_capacity, log_level):
|
|
||||||
super().__init__()
|
|
||||||
|
|
||||||
self.byte_capacity = byte_capacity
|
|
||||||
self.byte_count = 0
|
|
||||||
self.buffer = []
|
|
||||||
self.forgot = False
|
|
||||||
self.setLevel(log_level)
|
|
||||||
|
|
||||||
def emit(self, record):
|
|
||||||
message = record.getMessage() + '\n'
|
|
||||||
self.byte_count += len(message)
|
|
||||||
self.buffer.append(message)
|
|
||||||
|
|
||||||
if not self.byte_capacity:
|
|
||||||
return
|
|
||||||
|
|
||||||
while self.byte_count > self.byte_capacity and self.buffer:
|
|
||||||
self.byte_count -= len(self.buffer[0])
|
|
||||||
self.buffer.pop(0)
|
|
||||||
self.forgot = True
|
|
||||||
|
|
||||||
|
|
||||||
def format_buffered_logs_for_payload():
|
|
||||||
'''
|
|
||||||
Get the handler previously added to the root logger, and slurp buffered logs out of it to
|
|
||||||
send to Healthchecks.
|
|
||||||
'''
|
|
||||||
try:
|
|
||||||
buffering_handler = next(
|
|
||||||
handler
|
|
||||||
for handler in logging.getLogger().handlers
|
|
||||||
if isinstance(handler, Forgetful_buffering_handler)
|
|
||||||
)
|
|
||||||
except StopIteration:
|
|
||||||
# No handler means no payload.
|
|
||||||
return ''
|
|
||||||
|
|
||||||
payload = ''.join(message for message in buffering_handler.buffer)
|
|
||||||
|
|
||||||
if buffering_handler.forgot:
|
|
||||||
return PAYLOAD_TRUNCATION_INDICATOR + payload
|
|
||||||
|
|
||||||
return payload
|
|
||||||
|
|
||||||
|
|
||||||
def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
|
def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
|
||||||
|
@ -81,12 +30,14 @@ def initialize_monitor(hook_config, config, config_filename, monitoring_log_leve
|
||||||
|
|
||||||
ping_body_limit = max(
|
ping_body_limit = max(
|
||||||
hook_config.get('ping_body_limit', DEFAULT_PING_BODY_LIMIT_BYTES)
|
hook_config.get('ping_body_limit', DEFAULT_PING_BODY_LIMIT_BYTES)
|
||||||
- len(PAYLOAD_TRUNCATION_INDICATOR),
|
- len(borgmatic.hooks.logs.PAYLOAD_TRUNCATION_INDICATOR),
|
||||||
0,
|
0,
|
||||||
)
|
)
|
||||||
|
|
||||||
logging.getLogger().addHandler(
|
borgmatic.hooks.logs.add_handler(
|
||||||
Forgetful_buffering_handler(ping_body_limit, monitoring_log_level)
|
borgmatic.hooks.logs.Forgetful_buffering_handler(
|
||||||
|
HANDLER_IDENTIFIER, ping_body_limit, monitoring_log_level
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -109,15 +60,25 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
ping_url_is_uuid = re.search(r'\w{8}-\w{4}-\w{4}-\w{4}-\w{12}$', ping_url)
|
||||||
|
|
||||||
healthchecks_state = MONITOR_STATE_TO_HEALTHCHECKS.get(state)
|
healthchecks_state = MONITOR_STATE_TO_HEALTHCHECKS.get(state)
|
||||||
if healthchecks_state:
|
if healthchecks_state:
|
||||||
ping_url = f'{ping_url}/{healthchecks_state}'
|
ping_url = f'{ping_url}/{healthchecks_state}'
|
||||||
|
|
||||||
|
if hook_config.get('create_slug'):
|
||||||
|
if ping_url_is_uuid:
|
||||||
|
logger.warning(
|
||||||
|
f'{config_filename}: Healthchecks UUIDs do not support auto provisionning; ignoring'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
ping_url = f'{ping_url}?create=1'
|
||||||
|
|
||||||
logger.info(f'{config_filename}: Pinging Healthchecks {state.name.lower()}{dry_run_label}')
|
logger.info(f'{config_filename}: Pinging Healthchecks {state.name.lower()}{dry_run_label}')
|
||||||
logger.debug(f'{config_filename}: Using Healthchecks ping URL {ping_url}')
|
logger.debug(f'{config_filename}: Using Healthchecks ping URL {ping_url}')
|
||||||
|
|
||||||
if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG):
|
if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG):
|
||||||
payload = format_buffered_logs_for_payload()
|
payload = borgmatic.hooks.logs.format_buffered_logs_for_payload(HANDLER_IDENTIFIER)
|
||||||
else:
|
else:
|
||||||
payload = ''
|
payload = ''
|
||||||
|
|
||||||
|
@ -138,8 +99,4 @@ def destroy_monitor(hook_config, config, config_filename, monitoring_log_level,
|
||||||
Remove the monitor handler that was added to the root logger. This prevents the handler from
|
Remove the monitor handler that was added to the root logger. This prevents the handler from
|
||||||
getting reused by other instances of this monitor.
|
getting reused by other instances of this monitor.
|
||||||
'''
|
'''
|
||||||
logger = logging.getLogger()
|
borgmatic.hooks.logs.remove_handler(HANDLER_IDENTIFIER)
|
||||||
|
|
||||||
for handler in tuple(logger.handlers):
|
|
||||||
if isinstance(handler, Forgetful_buffering_handler):
|
|
||||||
logger.removeHandler(handler)
|
|
||||||
|
|
|
@ -0,0 +1,91 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
PAYLOAD_TRUNCATION_INDICATOR = '...\n'
|
||||||
|
|
||||||
|
|
||||||
|
class Forgetful_buffering_handler(logging.Handler):
|
||||||
|
'''
|
||||||
|
A buffering log handler that stores log messages in memory, and throws away messages (oldest
|
||||||
|
first) once a particular capacity in bytes is reached. But if the given byte capacity is zero,
|
||||||
|
don't throw away any messages.
|
||||||
|
|
||||||
|
The given identifier is used to distinguish the instance of this handler used for one monitoring
|
||||||
|
hook from those instances used for other monitoring hooks.
|
||||||
|
'''
|
||||||
|
|
||||||
|
def __init__(self, identifier, byte_capacity, log_level):
|
||||||
|
super().__init__()
|
||||||
|
|
||||||
|
self.identifier = identifier
|
||||||
|
self.byte_capacity = byte_capacity
|
||||||
|
self.byte_count = 0
|
||||||
|
self.buffer = []
|
||||||
|
self.forgot = False
|
||||||
|
self.setLevel(log_level)
|
||||||
|
|
||||||
|
def emit(self, record):
|
||||||
|
message = record.getMessage() + '\n'
|
||||||
|
self.byte_count += len(message)
|
||||||
|
self.buffer.append(message)
|
||||||
|
|
||||||
|
if not self.byte_capacity:
|
||||||
|
return
|
||||||
|
|
||||||
|
while self.byte_count > self.byte_capacity and self.buffer:
|
||||||
|
self.byte_count -= len(self.buffer[0])
|
||||||
|
self.buffer.pop(0)
|
||||||
|
self.forgot = True
|
||||||
|
|
||||||
|
|
||||||
|
def add_handler(handler): # pragma: no cover
|
||||||
|
'''
|
||||||
|
Add the given handler to the global logger.
|
||||||
|
'''
|
||||||
|
logging.getLogger().addHandler(handler)
|
||||||
|
|
||||||
|
|
||||||
|
def get_handler(identifier):
|
||||||
|
'''
|
||||||
|
Given the identifier for an existing Forgetful_buffering_handler instance, return the handler.
|
||||||
|
|
||||||
|
Raise ValueError if the handler isn't found.
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
return next(
|
||||||
|
handler
|
||||||
|
for handler in logging.getLogger().handlers
|
||||||
|
if isinstance(handler, Forgetful_buffering_handler) and handler.identifier == identifier
|
||||||
|
)
|
||||||
|
except StopIteration:
|
||||||
|
raise ValueError(f'A buffering handler for {identifier} was not found')
|
||||||
|
|
||||||
|
|
||||||
|
def format_buffered_logs_for_payload(identifier):
|
||||||
|
'''
|
||||||
|
Get the handler previously added to the root logger, and slurp buffered logs out of it to
|
||||||
|
send to Healthchecks.
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
buffering_handler = get_handler(identifier)
|
||||||
|
except ValueError:
|
||||||
|
# No handler means no payload.
|
||||||
|
return ''
|
||||||
|
|
||||||
|
payload = ''.join(message for message in buffering_handler.buffer)
|
||||||
|
|
||||||
|
if buffering_handler.forgot:
|
||||||
|
return PAYLOAD_TRUNCATION_INDICATOR + payload
|
||||||
|
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def remove_handler(identifier):
|
||||||
|
'''
|
||||||
|
Given the identifier for an existing Forgetful_buffering_handler instance, remove it.
|
||||||
|
'''
|
||||||
|
logger = logging.getLogger()
|
||||||
|
|
||||||
|
try:
|
||||||
|
logger.removeHandler(get_handler(identifier))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
|
@ -0,0 +1,154 @@
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import platform
|
||||||
|
import time
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from borgmatic.hooks import monitor
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
MONITOR_STATE_TO_LOKI = {
|
||||||
|
monitor.State.START: 'Started',
|
||||||
|
monitor.State.FINISH: 'Finished',
|
||||||
|
monitor.State.FAIL: 'Failed',
|
||||||
|
}
|
||||||
|
|
||||||
|
# Threshold at which logs get flushed to loki
|
||||||
|
MAX_BUFFER_LINES = 100
|
||||||
|
|
||||||
|
|
||||||
|
class Loki_log_buffer:
|
||||||
|
'''
|
||||||
|
A log buffer that allows to output the logs as loki requests in json. Allows
|
||||||
|
adding labels to the log stream and takes care of communication with loki.
|
||||||
|
'''
|
||||||
|
|
||||||
|
def __init__(self, url, dry_run):
|
||||||
|
self.url = url
|
||||||
|
self.dry_run = dry_run
|
||||||
|
self.root = {'streams': [{'stream': {}, 'values': []}]}
|
||||||
|
|
||||||
|
def add_value(self, value):
|
||||||
|
'''
|
||||||
|
Add a log entry to the stream.
|
||||||
|
'''
|
||||||
|
timestamp = str(time.time_ns())
|
||||||
|
self.root['streams'][0]['values'].append((timestamp, value))
|
||||||
|
|
||||||
|
def add_label(self, label, value):
|
||||||
|
'''
|
||||||
|
Add a label to the logging stream.
|
||||||
|
'''
|
||||||
|
self.root['streams'][0]['stream'][label] = value
|
||||||
|
|
||||||
|
def to_request(self):
|
||||||
|
return json.dumps(self.root)
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
'''
|
||||||
|
Gets the number of lines currently in the buffer.
|
||||||
|
'''
|
||||||
|
return len(self.root['streams'][0]['values'])
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
if self.dry_run:
|
||||||
|
# Just empty the buffer and skip
|
||||||
|
self.root['streams'][0]['values'] = []
|
||||||
|
logger.info('Skipped uploading logs to loki due to dry run')
|
||||||
|
return
|
||||||
|
|
||||||
|
if len(self) == 0:
|
||||||
|
# Skip as there are not logs to send yet
|
||||||
|
return
|
||||||
|
|
||||||
|
request_body = self.to_request()
|
||||||
|
self.root['streams'][0]['values'] = []
|
||||||
|
request_header = {'Content-Type': 'application/json'}
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = requests.post(self.url, headers=request_header, data=request_body, timeout=5)
|
||||||
|
result.raise_for_status()
|
||||||
|
except requests.RequestException:
|
||||||
|
logger.warning('Failed to upload logs to loki')
|
||||||
|
|
||||||
|
|
||||||
|
class Loki_log_handler(logging.Handler):
|
||||||
|
'''
|
||||||
|
A log handler that sends logs to loki.
|
||||||
|
'''
|
||||||
|
|
||||||
|
def __init__(self, url, dry_run):
|
||||||
|
super().__init__()
|
||||||
|
self.buffer = Loki_log_buffer(url, dry_run)
|
||||||
|
|
||||||
|
def emit(self, record):
|
||||||
|
'''
|
||||||
|
Add a log record from the logging module to the stream.
|
||||||
|
'''
|
||||||
|
self.raw(record.getMessage())
|
||||||
|
|
||||||
|
def add_label(self, key, value):
|
||||||
|
'''
|
||||||
|
Add a label to the logging stream.
|
||||||
|
'''
|
||||||
|
self.buffer.add_label(key, value)
|
||||||
|
|
||||||
|
def raw(self, msg):
|
||||||
|
'''
|
||||||
|
Add an arbitrary string as a log entry to the stream.
|
||||||
|
'''
|
||||||
|
self.buffer.add_value(msg)
|
||||||
|
|
||||||
|
if len(self.buffer) > MAX_BUFFER_LINES:
|
||||||
|
self.buffer.flush()
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
'''
|
||||||
|
Send the logs to loki and empty the buffer.
|
||||||
|
'''
|
||||||
|
self.buffer.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
|
||||||
|
'''
|
||||||
|
Add a handler to the root logger to regularly send the logs to loki.
|
||||||
|
'''
|
||||||
|
url = hook_config.get('url')
|
||||||
|
loki = Loki_log_handler(url, dry_run)
|
||||||
|
|
||||||
|
for key, value in hook_config.get('labels').items():
|
||||||
|
if value == '__hostname':
|
||||||
|
loki.add_label(key, platform.node())
|
||||||
|
elif value == '__config':
|
||||||
|
loki.add_label(key, os.path.basename(config_filename))
|
||||||
|
elif value == '__config_path':
|
||||||
|
loki.add_label(key, config_filename)
|
||||||
|
else:
|
||||||
|
loki.add_label(key, value)
|
||||||
|
|
||||||
|
logging.getLogger().addHandler(loki)
|
||||||
|
|
||||||
|
|
||||||
|
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
|
||||||
|
'''
|
||||||
|
Add an entry to the loki logger with the current state.
|
||||||
|
'''
|
||||||
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
|
if isinstance(handler, Loki_log_handler):
|
||||||
|
if state in MONITOR_STATE_TO_LOKI.keys():
|
||||||
|
handler.raw(f'{config_filename}: {MONITOR_STATE_TO_LOKI[state]} backup')
|
||||||
|
|
||||||
|
|
||||||
|
def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
|
||||||
|
'''
|
||||||
|
Remove the monitor handler that was added to the root logger.
|
||||||
|
'''
|
||||||
|
logger = logging.getLogger()
|
||||||
|
|
||||||
|
for handler in tuple(logger.handlers):
|
||||||
|
if isinstance(handler, Loki_log_handler):
|
||||||
|
handler.flush()
|
||||||
|
logger.removeHandler(handler)
|
|
@ -1,6 +1,7 @@
|
||||||
import copy
|
import copy
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import shlex
|
||||||
|
|
||||||
from borgmatic.execute import (
|
from borgmatic.execute import (
|
||||||
execute_command,
|
execute_command,
|
||||||
|
@ -16,7 +17,7 @@ def make_dump_path(config): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Make the dump path from the given configuration dict and the name of this hook.
|
Make the dump path from the given configuration dict and the name of this hook.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_path(
|
return dump.make_data_source_dump_path(
|
||||||
config.get('borgmatic_source_directory'), 'mariadb_databases'
|
config.get('borgmatic_source_directory'), 'mariadb_databases'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -35,8 +36,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
|
mariadb_show_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb')
|
||||||
|
)
|
||||||
show_command = (
|
show_command = (
|
||||||
('mariadb',)
|
mariadb_show_command
|
||||||
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
|
@ -62,24 +66,29 @@ def execute_dump_command(
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named
|
Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named
|
||||||
pipe constructed from the given dump path and database names. Use the given log prefix in any
|
pipe constructed from the given dump path and database name. Use the given log prefix in any
|
||||||
log entries.
|
log entries.
|
||||||
|
|
||||||
Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
|
Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
|
||||||
this is a dry run, then don't actually dump anything and return None.
|
this is a dry run, then don't actually dump anything and return None.
|
||||||
'''
|
'''
|
||||||
database_name = database['name']
|
database_name = database['name']
|
||||||
dump_filename = dump.make_database_dump_filename(
|
dump_filename = dump.make_data_source_dump_filename(
|
||||||
dump_path, database['name'], database.get('hostname')
|
dump_path, database['name'], database.get('hostname')
|
||||||
)
|
)
|
||||||
|
|
||||||
if os.path.exists(dump_filename):
|
if os.path.exists(dump_filename):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f'{log_prefix}: Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}'
|
f'{log_prefix}: Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}'
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
mariadb_dump_command = tuple(
|
||||||
|
shlex.quote(part)
|
||||||
|
for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump')
|
||||||
|
)
|
||||||
dump_command = (
|
dump_command = (
|
||||||
('mariadb-dump',)
|
mariadb_dump_command
|
||||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
||||||
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
|
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
|
@ -106,7 +115,15 @@ def execute_dump_command(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def dump_databases(databases, config, log_prefix, dry_run):
|
def use_streaming(databases, config, log_prefix):
|
||||||
|
'''
|
||||||
|
Given a sequence of MariaDB database configuration dicts, a configuration dict (ignored), and a
|
||||||
|
log prefix (ignored), return whether streaming will be using during dumps.
|
||||||
|
'''
|
||||||
|
return any(databases)
|
||||||
|
|
||||||
|
|
||||||
|
def dump_data_sources(databases, config, log_prefix, dry_run):
|
||||||
'''
|
'''
|
||||||
Dump the given MariaDB databases to a named pipe. The databases are supplied as a sequence of
|
Dump the given MariaDB databases to a named pipe. The databases are supplied as a sequence of
|
||||||
dicts, one dict describing each database as per the configuration schema. Use the given
|
dicts, one dict describing each database as per the configuration schema. Use the given
|
||||||
|
@ -165,49 +182,59 @@ def dump_databases(databases, config, log_prefix, dry_run):
|
||||||
return [process for process in processes if process]
|
return [process for process in processes if process]
|
||||||
|
|
||||||
|
|
||||||
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Remove all database dump files for this hook regardless of the given databases. Use the given
|
Remove all database dump files for this hook regardless of the given databases. Use the given
|
||||||
configuration dict to construct the destination path and the log prefix in any log entries. If
|
configuration dict to construct the destination path and the log prefix in any log entries. If
|
||||||
this is a dry run, then don't actually remove anything.
|
this is a dry run, then don't actually remove anything.
|
||||||
'''
|
'''
|
||||||
dump.remove_database_dumps(make_dump_path(config), 'MariaDB', log_prefix, dry_run)
|
dump.remove_data_source_dumps(make_dump_path(config), 'MariaDB', log_prefix, dry_run)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
|
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
|
||||||
database name to match, return the corresponding glob patterns to match the database dump in an
|
database name to match, return the corresponding glob patterns to match the database dump in an
|
||||||
archive.
|
archive.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*')
|
return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*')
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dump(
|
def restore_data_source_dump(
|
||||||
hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
|
hook_config, config, log_prefix, data_source, dry_run, extract_process, connection_params
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Restore a database from the given extract stream. The database is supplied as a configuration
|
Restore a database from the given extract stream. The database is supplied as a data source
|
||||||
dict, but the given hook configuration is ignored. The given configuration dict is used to
|
configuration dict, but the given hook configuration is ignored. The given configuration dict is
|
||||||
construct the destination path, and the given log prefix is used for any log entries. If this is
|
used to construct the destination path, and the given log prefix is used for any log entries. If
|
||||||
a dry run, then don't actually restore anything. Trigger the given active extract process (an
|
this is a dry run, then don't actually restore anything. Trigger the given active extract
|
||||||
instance of subprocess.Popen) to produce output to consume.
|
process (an instance of subprocess.Popen) to produce output to consume.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
||||||
hostname = connection_params['hostname'] or database.get(
|
hostname = connection_params['hostname'] or data_source.get(
|
||||||
'restore_hostname', database.get('hostname')
|
'restore_hostname', data_source.get('hostname')
|
||||||
)
|
)
|
||||||
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
|
port = str(
|
||||||
username = connection_params['username'] or database.get(
|
connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
|
||||||
'restore_username', database.get('username')
|
|
||||||
)
|
)
|
||||||
password = connection_params['password'] or database.get(
|
username = connection_params['username'] or data_source.get(
|
||||||
'restore_password', database.get('password')
|
'restore_username', data_source.get('username')
|
||||||
|
)
|
||||||
|
password = connection_params['password'] or data_source.get(
|
||||||
|
'restore_password', data_source.get('password')
|
||||||
)
|
)
|
||||||
|
|
||||||
|
mariadb_restore_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb')
|
||||||
|
)
|
||||||
restore_command = (
|
restore_command = (
|
||||||
('mariadb', '--batch')
|
mariadb_restore_command
|
||||||
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
+ ('--batch',)
|
||||||
|
+ (
|
||||||
|
tuple(data_source['restore_options'].split(' '))
|
||||||
|
if 'restore_options' in data_source
|
||||||
|
else ()
|
||||||
|
)
|
||||||
+ (('--host', hostname) if hostname else ())
|
+ (('--host', hostname) if hostname else ())
|
||||||
+ (('--port', str(port)) if port else ())
|
+ (('--port', str(port)) if port else ())
|
||||||
+ (('--protocol', 'tcp') if hostname or port else ())
|
+ (('--protocol', 'tcp') if hostname or port else ())
|
||||||
|
@ -215,7 +242,7 @@ def restore_database_dump(
|
||||||
)
|
)
|
||||||
extra_environment = {'MYSQL_PWD': password} if password else None
|
extra_environment = {'MYSQL_PWD': password} if password else None
|
||||||
|
|
||||||
logger.debug(f"{log_prefix}: Restoring MariaDB database {database['name']}{dry_run_label}")
|
logger.debug(f"{log_prefix}: Restoring MariaDB database {data_source['name']}{dry_run_label}")
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import logging
|
import logging
|
||||||
|
import shlex
|
||||||
|
|
||||||
from borgmatic.execute import execute_command, execute_command_with_processes
|
from borgmatic.execute import execute_command, execute_command_with_processes
|
||||||
from borgmatic.hooks import dump
|
from borgmatic.hooks import dump
|
||||||
|
@ -10,12 +11,20 @@ def make_dump_path(config): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Make the dump path from the given configuration dict and the name of this hook.
|
Make the dump path from the given configuration dict and the name of this hook.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_path(
|
return dump.make_data_source_dump_path(
|
||||||
config.get('borgmatic_source_directory'), 'mongodb_databases'
|
config.get('borgmatic_source_directory'), 'mongodb_databases'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def dump_databases(databases, config, log_prefix, dry_run):
|
def use_streaming(databases, config, log_prefix):
|
||||||
|
'''
|
||||||
|
Given a sequence of MongoDB database configuration dicts, a configuration dict (ignored), and a
|
||||||
|
log prefix (ignored), return whether streaming will be using during dumps.
|
||||||
|
'''
|
||||||
|
return any(database.get('format') != 'directory' for database in databases)
|
||||||
|
|
||||||
|
|
||||||
|
def dump_data_sources(databases, config, log_prefix, dry_run):
|
||||||
'''
|
'''
|
||||||
Dump the given MongoDB databases to a named pipe. The databases are supplied as a sequence of
|
Dump the given MongoDB databases to a named pipe. The databases are supplied as a sequence of
|
||||||
dicts, one dict describing each database as per the configuration schema. Use the configuration
|
dicts, one dict describing each database as per the configuration schema. Use the configuration
|
||||||
|
@ -31,7 +40,7 @@ def dump_databases(databases, config, log_prefix, dry_run):
|
||||||
processes = []
|
processes = []
|
||||||
for database in databases:
|
for database in databases:
|
||||||
name = database['name']
|
name = database['name']
|
||||||
dump_filename = dump.make_database_dump_filename(
|
dump_filename = dump.make_data_source_dump_filename(
|
||||||
make_dump_path(config), name, database.get('hostname')
|
make_dump_path(config), name, database.get('hostname')
|
||||||
)
|
)
|
||||||
dump_format = database.get('format', 'archive')
|
dump_format = database.get('format', 'archive')
|
||||||
|
@ -62,62 +71,66 @@ def build_dump_command(database, dump_filename, dump_format):
|
||||||
|
|
||||||
return (
|
return (
|
||||||
('mongodump',)
|
('mongodump',)
|
||||||
+ (('--out', dump_filename) if dump_format == 'directory' else ())
|
+ (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
+ (('--username', shlex.quote(database['username'])) if 'username' in database else ())
|
||||||
+ (('--password', database['password']) if 'password' in database else ())
|
+ (('--password', shlex.quote(database['password'])) if 'password' in database else ())
|
||||||
+ (
|
+ (
|
||||||
('--authenticationDatabase', database['authentication_database'])
|
('--authenticationDatabase', shlex.quote(database['authentication_database']))
|
||||||
if 'authentication_database' in database
|
if 'authentication_database' in database
|
||||||
else ()
|
else ()
|
||||||
)
|
)
|
||||||
+ (('--db', database['name']) if not all_databases else ())
|
+ (('--db', shlex.quote(database['name'])) if not all_databases else ())
|
||||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
+ (
|
||||||
+ (('--archive', '>', dump_filename) if dump_format != 'directory' else ())
|
tuple(shlex.quote(option) for option in database['options'].split(' '))
|
||||||
|
if 'options' in database
|
||||||
|
else ()
|
||||||
|
)
|
||||||
|
+ (('--archive', '>', shlex.quote(dump_filename)) if dump_format != 'directory' else ())
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Remove all database dump files for this hook regardless of the given databases. Use the log
|
Remove all database dump files for this hook regardless of the given databases. Use the log
|
||||||
prefix in any log entries. Use the given configuration dict to construct the destination path.
|
prefix in any log entries. Use the given configuration dict to construct the destination path.
|
||||||
If this is a dry run, then don't actually remove anything.
|
If this is a dry run, then don't actually remove anything.
|
||||||
'''
|
'''
|
||||||
dump.remove_database_dumps(make_dump_path(config), 'MongoDB', log_prefix, dry_run)
|
dump.remove_data_source_dumps(make_dump_path(config), 'MongoDB', log_prefix, dry_run)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Given a sequence of database configurations dicts, a configuration dict, a prefix to log with,
|
Given a sequence of database configurations dicts, a configuration dict, a prefix to log with,
|
||||||
and a database name to match, return the corresponding glob patterns to match the database dump
|
and a database name to match, return the corresponding glob patterns to match the database dump
|
||||||
in an archive.
|
in an archive.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*')
|
return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*')
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dump(
|
def restore_data_source_dump(
|
||||||
hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
|
hook_config, config, log_prefix, data_source, dry_run, extract_process, connection_params
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Restore a database from the given extract stream. The database is supplied as a configuration
|
Restore a database from the given extract stream. The database is supplied as a data source
|
||||||
dict, but the given hook configuration is ignored. The given configuration dict is used to
|
configuration dict, but the given hook configuration is ignored. The given configuration dict is
|
||||||
construct the destination path, and the given log prefix is used for any log entries. If this is
|
used to construct the destination path, and the given log prefix is used for any log entries. If
|
||||||
a dry run, then don't actually restore anything. Trigger the given active extract process (an
|
this is a dry run, then don't actually restore anything. Trigger the given active extract
|
||||||
instance of subprocess.Popen) to produce output to consume.
|
process (an instance of subprocess.Popen) to produce output to consume.
|
||||||
|
|
||||||
If the extract process is None, then restore the dump from the filesystem rather than from an
|
If the extract process is None, then restore the dump from the filesystem rather than from an
|
||||||
extract stream.
|
extract stream.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
||||||
dump_filename = dump.make_database_dump_filename(
|
dump_filename = dump.make_data_source_dump_filename(
|
||||||
make_dump_path(config), database['name'], database.get('hostname')
|
make_dump_path(config), data_source['name'], data_source.get('hostname')
|
||||||
)
|
)
|
||||||
restore_command = build_restore_command(
|
restore_command = build_restore_command(
|
||||||
extract_process, database, dump_filename, connection_params
|
extract_process, data_source, dump_filename, connection_params
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}")
|
logger.debug(f"{log_prefix}: Restoring MongoDB database {data_source['name']}{dry_run_label}")
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
|
|
||||||
MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy')
|
MONITOR_HOOK_NAMES = ('apprise', 'healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy', 'loki')
|
||||||
|
|
||||||
|
|
||||||
class State(Enum):
|
class State(Enum):
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import copy
|
import copy
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import shlex
|
||||||
|
|
||||||
from borgmatic.execute import (
|
from borgmatic.execute import (
|
||||||
execute_command,
|
execute_command,
|
||||||
|
@ -16,7 +17,9 @@ def make_dump_path(config): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Make the dump path from the given configuration dict and the name of this hook.
|
Make the dump path from the given configuration dict and the name of this hook.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_path(config.get('borgmatic_source_directory'), 'mysql_databases')
|
return dump.make_data_source_dump_path(
|
||||||
|
config.get('borgmatic_source_directory'), 'mysql_databases'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
|
SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
|
||||||
|
@ -33,8 +36,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
|
mysql_show_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql')
|
||||||
|
)
|
||||||
show_command = (
|
show_command = (
|
||||||
('mysql',)
|
mysql_show_command
|
||||||
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
|
@ -60,24 +66,28 @@ def execute_dump_command(
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a
|
Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a
|
||||||
named pipe constructed from the given dump path and database names. Use the given log prefix in
|
named pipe constructed from the given dump path and database name. Use the given log prefix in
|
||||||
any log entries.
|
any log entries.
|
||||||
|
|
||||||
Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
|
Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
|
||||||
this is a dry run, then don't actually dump anything and return None.
|
this is a dry run, then don't actually dump anything and return None.
|
||||||
'''
|
'''
|
||||||
database_name = database['name']
|
database_name = database['name']
|
||||||
dump_filename = dump.make_database_dump_filename(
|
dump_filename = dump.make_data_source_dump_filename(
|
||||||
dump_path, database['name'], database.get('hostname')
|
dump_path, database['name'], database.get('hostname')
|
||||||
)
|
)
|
||||||
|
|
||||||
if os.path.exists(dump_filename):
|
if os.path.exists(dump_filename):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f'{log_prefix}: Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}'
|
f'{log_prefix}: Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}'
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
mysql_dump_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump')
|
||||||
|
)
|
||||||
dump_command = (
|
dump_command = (
|
||||||
('mysqldump',)
|
mysql_dump_command
|
||||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
||||||
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
|
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
|
@ -104,7 +114,15 @@ def execute_dump_command(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def dump_databases(databases, config, log_prefix, dry_run):
|
def use_streaming(databases, config, log_prefix):
|
||||||
|
'''
|
||||||
|
Given a sequence of MySQL database configuration dicts, a configuration dict (ignored), and a
|
||||||
|
log prefix (ignored), return whether streaming will be using during dumps.
|
||||||
|
'''
|
||||||
|
return any(databases)
|
||||||
|
|
||||||
|
|
||||||
|
def dump_data_sources(databases, config, log_prefix, dry_run):
|
||||||
'''
|
'''
|
||||||
Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence
|
Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence
|
||||||
of dicts, one dict describing each database as per the configuration schema. Use the given
|
of dicts, one dict describing each database as per the configuration schema. Use the given
|
||||||
|
@ -162,49 +180,59 @@ def dump_databases(databases, config, log_prefix, dry_run):
|
||||||
return [process for process in processes if process]
|
return [process for process in processes if process]
|
||||||
|
|
||||||
|
|
||||||
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Remove all database dump files for this hook regardless of the given databases. Use the given
|
Remove all database dump files for this hook regardless of the given databases. Use the given
|
||||||
configuration dict to construct the destination path and the log prefix in any log entries. If
|
configuration dict to construct the destination path and the log prefix in any log entries. If
|
||||||
this is a dry run, then don't actually remove anything.
|
this is a dry run, then don't actually remove anything.
|
||||||
'''
|
'''
|
||||||
dump.remove_database_dumps(make_dump_path(config), 'MySQL', log_prefix, dry_run)
|
dump.remove_data_source_dumps(make_dump_path(config), 'MySQL', log_prefix, dry_run)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
|
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
|
||||||
database name to match, return the corresponding glob patterns to match the database dump in an
|
database name to match, return the corresponding glob patterns to match the database dump in an
|
||||||
archive.
|
archive.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*')
|
return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*')
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dump(
|
def restore_data_source_dump(
|
||||||
hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
|
hook_config, config, log_prefix, data_source, dry_run, extract_process, connection_params
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Restore a database from the given extract stream. The database is supplied as a configuration
|
Restore a database from the given extract stream. The database is supplied as a data source
|
||||||
dict, but the given hook configuration is ignored. The given configuration dict is used to
|
configuration dict, but the given hook configuration is ignored. The given configuration dict is
|
||||||
construct the destination path, and the given log prefix is used for any log entries. If this is
|
used to construct the destination path, and the given log prefix is used for any log entries. If
|
||||||
a dry run, then don't actually restore anything. Trigger the given active extract process (an
|
this is a dry run, then don't actually restore anything. Trigger the given active extract
|
||||||
instance of subprocess.Popen) to produce output to consume.
|
process (an instance of subprocess.Popen) to produce output to consume.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
||||||
hostname = connection_params['hostname'] or database.get(
|
hostname = connection_params['hostname'] or data_source.get(
|
||||||
'restore_hostname', database.get('hostname')
|
'restore_hostname', data_source.get('hostname')
|
||||||
)
|
)
|
||||||
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
|
port = str(
|
||||||
username = connection_params['username'] or database.get(
|
connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
|
||||||
'restore_username', database.get('username')
|
|
||||||
)
|
)
|
||||||
password = connection_params['password'] or database.get(
|
username = connection_params['username'] or data_source.get(
|
||||||
'restore_password', database.get('password')
|
'restore_username', data_source.get('username')
|
||||||
|
)
|
||||||
|
password = connection_params['password'] or data_source.get(
|
||||||
|
'restore_password', data_source.get('password')
|
||||||
)
|
)
|
||||||
|
|
||||||
|
mysql_restore_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql')
|
||||||
|
)
|
||||||
restore_command = (
|
restore_command = (
|
||||||
('mysql', '--batch')
|
mysql_restore_command
|
||||||
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
+ ('--batch',)
|
||||||
|
+ (
|
||||||
|
tuple(data_source['restore_options'].split(' '))
|
||||||
|
if 'restore_options' in data_source
|
||||||
|
else ()
|
||||||
|
)
|
||||||
+ (('--host', hostname) if hostname else ())
|
+ (('--host', hostname) if hostname else ())
|
||||||
+ (('--port', str(port)) if port else ())
|
+ (('--port', str(port)) if port else ())
|
||||||
+ (('--protocol', 'tcp') if hostname or port else ())
|
+ (('--protocol', 'tcp') if hostname or port else ())
|
||||||
|
@ -212,7 +240,7 @@ def restore_database_dump(
|
||||||
)
|
)
|
||||||
extra_environment = {'MYSQL_PWD': password} if password else None
|
extra_environment = {'MYSQL_PWD': password} if password else None
|
||||||
|
|
||||||
logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}")
|
logger.debug(f"{log_prefix}: Restoring MySQL database {data_source['name']}{dry_run_label}")
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
|
@ -50,9 +50,16 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
|
||||||
|
|
||||||
username = hook_config.get('username')
|
username = hook_config.get('username')
|
||||||
password = hook_config.get('password')
|
password = hook_config.get('password')
|
||||||
|
access_token = hook_config.get('access_token')
|
||||||
auth = None
|
auth = None
|
||||||
if (username and password) is not None:
|
|
||||||
|
if access_token is not None:
|
||||||
|
if username or password:
|
||||||
|
logger.warning(
|
||||||
|
f'{config_filename}: ntfy access_token is set but so is username/password, only using access_token'
|
||||||
|
)
|
||||||
|
auth = requests.auth.HTTPBasicAuth('', access_token)
|
||||||
|
elif (username and password) is not None:
|
||||||
auth = requests.auth.HTTPBasicAuth(username, password)
|
auth = requests.auth.HTTPBasicAuth(username, password)
|
||||||
logger.info(f'{config_filename}: Using basic auth with user {username} for ntfy')
|
logger.info(f'{config_filename}: Using basic auth with user {username} for ntfy')
|
||||||
elif username is not None:
|
elif username is not None:
|
||||||
|
|
|
@ -18,15 +18,15 @@ def make_dump_path(config): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Make the dump path from the given configuration dict and the name of this hook.
|
Make the dump path from the given configuration dict and the name of this hook.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_path(
|
return dump.make_data_source_dump_path(
|
||||||
config.get('borgmatic_source_directory'), 'postgresql_databases'
|
config.get('borgmatic_source_directory'), 'postgresql_databases'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def make_extra_environment(database, restore_connection_params=None):
|
def make_extra_environment(database, restore_connection_params=None):
|
||||||
'''
|
'''
|
||||||
Make the extra_environment dict from the given database configuration.
|
Make the extra_environment dict from the given database configuration. If restore connection
|
||||||
If restore connection params are given, this is for a restore operation.
|
params are given, this is for a restore operation.
|
||||||
'''
|
'''
|
||||||
extra = dict()
|
extra = dict()
|
||||||
|
|
||||||
|
@ -40,7 +40,8 @@ def make_extra_environment(database, restore_connection_params=None):
|
||||||
except (AttributeError, KeyError):
|
except (AttributeError, KeyError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
|
if 'ssl_mode' in database:
|
||||||
|
extra['PGSSLMODE'] = database['ssl_mode']
|
||||||
if 'ssl_cert' in database:
|
if 'ssl_cert' in database:
|
||||||
extra['PGSSLCERT'] = database['ssl_cert']
|
extra['PGSSLCERT'] = database['ssl_cert']
|
||||||
if 'ssl_key' in database:
|
if 'ssl_key' in database:
|
||||||
|
@ -49,6 +50,7 @@ def make_extra_environment(database, restore_connection_params=None):
|
||||||
extra['PGSSLROOTCERT'] = database['ssl_root_cert']
|
extra['PGSSLROOTCERT'] = database['ssl_root_cert']
|
||||||
if 'ssl_crl' in database:
|
if 'ssl_crl' in database:
|
||||||
extra['PGSSLCRL'] = database['ssl_crl']
|
extra['PGSSLCRL'] = database['ssl_crl']
|
||||||
|
|
||||||
return extra
|
return extra
|
||||||
|
|
||||||
|
|
||||||
|
@ -71,9 +73,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
psql_command = shlex.split(database.get('psql_command') or 'psql')
|
psql_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql')
|
||||||
|
)
|
||||||
list_command = (
|
list_command = (
|
||||||
tuple(psql_command)
|
psql_command
|
||||||
+ ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
|
+ ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
|
@ -92,7 +96,15 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def dump_databases(databases, config, log_prefix, dry_run):
|
def use_streaming(databases, config, log_prefix):
|
||||||
|
'''
|
||||||
|
Given a sequence of PostgreSQL database configuration dicts, a configuration dict (ignored), and
|
||||||
|
a log prefix (ignored), return whether streaming will be using during dumps.
|
||||||
|
'''
|
||||||
|
return any(database.get('format') != 'directory' for database in databases)
|
||||||
|
|
||||||
|
|
||||||
|
def dump_data_sources(databases, config, log_prefix, dry_run):
|
||||||
'''
|
'''
|
||||||
Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
|
Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
|
||||||
dicts, one dict describing each database as per the configuration schema. Use the given
|
dicts, one dict describing each database as per the configuration schema. Use the given
|
||||||
|
@ -125,8 +137,11 @@ def dump_databases(databases, config, log_prefix, dry_run):
|
||||||
for database_name in dump_database_names:
|
for database_name in dump_database_names:
|
||||||
dump_format = database.get('format', None if database_name == 'all' else 'custom')
|
dump_format = database.get('format', None if database_name == 'all' else 'custom')
|
||||||
default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump'
|
default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump'
|
||||||
dump_command = database.get('pg_dump_command') or default_dump_command
|
dump_command = tuple(
|
||||||
dump_filename = dump.make_database_dump_filename(
|
shlex.quote(part)
|
||||||
|
for part in shlex.split(database.get('pg_dump_command') or default_dump_command)
|
||||||
|
)
|
||||||
|
dump_filename = dump.make_data_source_dump_filename(
|
||||||
dump_path, database_name, database.get('hostname')
|
dump_path, database_name, database.get('hostname')
|
||||||
)
|
)
|
||||||
if os.path.exists(dump_filename):
|
if os.path.exists(dump_filename):
|
||||||
|
@ -136,24 +151,32 @@ def dump_databases(databases, config, log_prefix, dry_run):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
command = (
|
command = (
|
||||||
(
|
dump_command
|
||||||
dump_command,
|
+ (
|
||||||
'--no-password',
|
'--no-password',
|
||||||
'--clean',
|
'--clean',
|
||||||
'--if-exists',
|
'--if-exists',
|
||||||
)
|
)
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
+ (
|
||||||
|
('--username', shlex.quote(database['username']))
|
||||||
|
if 'username' in database
|
||||||
|
else ()
|
||||||
|
)
|
||||||
+ (('--no-owner',) if database.get('no_owner', False) else ())
|
+ (('--no-owner',) if database.get('no_owner', False) else ())
|
||||||
+ (('--format', dump_format) if dump_format else ())
|
+ (('--format', shlex.quote(dump_format)) if dump_format else ())
|
||||||
+ (('--file', dump_filename) if dump_format == 'directory' else ())
|
+ (('--file', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
|
||||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
+ (
|
||||||
+ (() if database_name == 'all' else (database_name,))
|
tuple(shlex.quote(option) for option in database['options'].split(' '))
|
||||||
|
if 'options' in database
|
||||||
|
else ()
|
||||||
|
)
|
||||||
|
+ (() if database_name == 'all' else (shlex.quote(database_name),))
|
||||||
# Use shell redirection rather than the --file flag to sidestep synchronization issues
|
# Use shell redirection rather than the --file flag to sidestep synchronization issues
|
||||||
# when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump
|
# when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump
|
||||||
# format in a particular, a named destination is required, and redirection doesn't work.
|
# format in a particular, a named destination is required, and redirection doesn't work.
|
||||||
+ (('>', dump_filename) if dump_format != 'directory' else ())
|
+ (('>', shlex.quote(dump_filename)) if dump_format != 'directory' else ())
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
@ -183,33 +206,33 @@ def dump_databases(databases, config, log_prefix, dry_run):
|
||||||
return processes
|
return processes
|
||||||
|
|
||||||
|
|
||||||
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Remove all database dump files for this hook regardless of the given databases. Use the given
|
Remove all database dump files for this hook regardless of the given databases. Use the given
|
||||||
configuration dict to construct the destination path and the log prefix in any log entries. If
|
configuration dict to construct the destination path and the log prefix in any log entries. If
|
||||||
this is a dry run, then don't actually remove anything.
|
this is a dry run, then don't actually remove anything.
|
||||||
'''
|
'''
|
||||||
dump.remove_database_dumps(make_dump_path(config), 'PostgreSQL', log_prefix, dry_run)
|
dump.remove_data_source_dumps(make_dump_path(config), 'PostgreSQL', log_prefix, dry_run)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
|
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
|
||||||
database name to match, return the corresponding glob patterns to match the database dump in an
|
database name to match, return the corresponding glob patterns to match the database dump in an
|
||||||
archive.
|
archive.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*')
|
return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*')
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dump(
|
def restore_data_source_dump(
|
||||||
hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
|
hook_config, config, log_prefix, data_source, dry_run, extract_process, connection_params
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Restore a database from the given extract stream. The database is supplied as a configuration
|
Restore a database from the given extract stream. The database is supplied as a data source
|
||||||
dict, but the given hook configuration is ignored. The given configuration dict is used to
|
configuration dict, but the given hook configuration is ignored. The given configuration dict is
|
||||||
construct the destination path, and the given log prefix is used for any log entries. If this is
|
used to construct the destination path, and the given log prefix is used for any log entries. If
|
||||||
a dry run, then don't actually restore anything. Trigger the given active extract process (an
|
this is a dry run, then don't actually restore anything. Trigger the given active extract
|
||||||
instance of subprocess.Popen) to produce output to consume.
|
process (an instance of subprocess.Popen) to produce output to consume.
|
||||||
|
|
||||||
If the extract process is None, then restore the dump from the filesystem rather than from an
|
If the extract process is None, then restore the dump from the filesystem rather than from an
|
||||||
extract stream.
|
extract stream.
|
||||||
|
@ -218,54 +241,71 @@ def restore_database_dump(
|
||||||
hostname, port, username, and password.
|
hostname, port, username, and password.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
||||||
hostname = connection_params['hostname'] or database.get(
|
hostname = connection_params['hostname'] or data_source.get(
|
||||||
'restore_hostname', database.get('hostname')
|
'restore_hostname', data_source.get('hostname')
|
||||||
)
|
)
|
||||||
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
|
port = str(
|
||||||
username = connection_params['username'] or database.get(
|
connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
|
||||||
'restore_username', database.get('username')
|
)
|
||||||
|
username = connection_params['username'] or data_source.get(
|
||||||
|
'restore_username', data_source.get('username')
|
||||||
)
|
)
|
||||||
|
|
||||||
all_databases = bool(database['name'] == 'all')
|
all_databases = bool(data_source['name'] == 'all')
|
||||||
dump_filename = dump.make_database_dump_filename(
|
dump_filename = dump.make_data_source_dump_filename(
|
||||||
make_dump_path(config), database['name'], database.get('hostname')
|
make_dump_path(config), data_source['name'], data_source.get('hostname')
|
||||||
|
)
|
||||||
|
psql_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql')
|
||||||
)
|
)
|
||||||
psql_command = shlex.split(database.get('psql_command') or 'psql')
|
|
||||||
analyze_command = (
|
analyze_command = (
|
||||||
tuple(psql_command)
|
psql_command
|
||||||
+ ('--no-password', '--no-psqlrc', '--quiet')
|
+ ('--no-password', '--no-psqlrc', '--quiet')
|
||||||
+ (('--host', hostname) if hostname else ())
|
+ (('--host', hostname) if hostname else ())
|
||||||
+ (('--port', port) if port else ())
|
+ (('--port', port) if port else ())
|
||||||
+ (('--username', username) if username else ())
|
+ (('--username', username) if username else ())
|
||||||
+ (('--dbname', database['name']) if not all_databases else ())
|
+ (('--dbname', data_source['name']) if not all_databases else ())
|
||||||
+ (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
|
+ (
|
||||||
|
tuple(data_source['analyze_options'].split(' '))
|
||||||
|
if 'analyze_options' in data_source
|
||||||
|
else ()
|
||||||
|
)
|
||||||
+ ('--command', 'ANALYZE')
|
+ ('--command', 'ANALYZE')
|
||||||
)
|
)
|
||||||
use_psql_command = all_databases or database.get('format') == 'plain'
|
use_psql_command = all_databases or data_source.get('format') == 'plain'
|
||||||
pg_restore_command = shlex.split(database.get('pg_restore_command') or 'pg_restore')
|
pg_restore_command = tuple(
|
||||||
|
shlex.quote(part)
|
||||||
|
for part in shlex.split(data_source.get('pg_restore_command') or 'pg_restore')
|
||||||
|
)
|
||||||
restore_command = (
|
restore_command = (
|
||||||
tuple(psql_command if use_psql_command else pg_restore_command)
|
(psql_command if use_psql_command else pg_restore_command)
|
||||||
+ ('--no-password',)
|
+ ('--no-password',)
|
||||||
+ (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
|
+ (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
|
||||||
+ (('--dbname', database['name']) if not all_databases else ())
|
+ (('--dbname', data_source['name']) if not all_databases else ())
|
||||||
+ (('--host', hostname) if hostname else ())
|
+ (('--host', hostname) if hostname else ())
|
||||||
+ (('--port', port) if port else ())
|
+ (('--port', port) if port else ())
|
||||||
+ (('--username', username) if username else ())
|
+ (('--username', username) if username else ())
|
||||||
+ (('--no-owner',) if database.get('no_owner', False) else ())
|
+ (('--no-owner',) if data_source.get('no_owner', False) else ())
|
||||||
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
+ (
|
||||||
|
tuple(data_source['restore_options'].split(' '))
|
||||||
|
if 'restore_options' in data_source
|
||||||
|
else ()
|
||||||
|
)
|
||||||
+ (() if extract_process else (dump_filename,))
|
+ (() if extract_process else (dump_filename,))
|
||||||
+ tuple(
|
+ tuple(
|
||||||
itertools.chain.from_iterable(('--schema', schema) for schema in database['schemas'])
|
itertools.chain.from_iterable(('--schema', schema) for schema in data_source['schemas'])
|
||||||
if database.get('schemas')
|
if data_source.get('schemas')
|
||||||
else ()
|
else ()
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
extra_environment = make_extra_environment(
|
extra_environment = make_extra_environment(
|
||||||
database, restore_connection_params=connection_params
|
data_source, restore_connection_params=connection_params
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")
|
logger.debug(
|
||||||
|
f"{log_prefix}: Restoring PostgreSQL database {data_source['name']}{dry_run_label}"
|
||||||
|
)
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import shlex
|
||||||
|
|
||||||
from borgmatic.execute import execute_command, execute_command_with_processes
|
from borgmatic.execute import execute_command, execute_command_with_processes
|
||||||
from borgmatic.hooks import dump
|
from borgmatic.hooks import dump
|
||||||
|
@ -11,17 +12,27 @@ def make_dump_path(config): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Make the dump path from the given configuration dict and the name of this hook.
|
Make the dump path from the given configuration dict and the name of this hook.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_path(
|
return dump.make_data_source_dump_path(
|
||||||
config.get('borgmatic_source_directory'), 'sqlite_databases'
|
config.get('borgmatic_source_directory'), 'sqlite_databases'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def dump_databases(databases, config, log_prefix, dry_run):
|
def use_streaming(databases, config, log_prefix):
|
||||||
'''
|
'''
|
||||||
Dump the given SQLite3 databases to a file. The databases are supplied as a sequence of
|
Given a sequence of SQLite database configuration dicts, a configuration dict (ignored), and a
|
||||||
|
log prefix (ignored), return whether streaming will be using during dumps.
|
||||||
|
'''
|
||||||
|
return any(databases)
|
||||||
|
|
||||||
|
|
||||||
|
def dump_data_sources(databases, config, log_prefix, dry_run):
|
||||||
|
'''
|
||||||
|
Dump the given SQLite databases to a named pipe. The databases are supplied as a sequence of
|
||||||
configuration dicts, as per the configuration schema. Use the given configuration dict to
|
configuration dicts, as per the configuration schema. Use the given configuration dict to
|
||||||
construct the destination path and the given log prefix in any log entries. If this is a dry
|
construct the destination path and the given log prefix in any log entries.
|
||||||
run, then don't actually dump anything.
|
|
||||||
|
Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
|
||||||
|
pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
|
||||||
processes = []
|
processes = []
|
||||||
|
@ -39,7 +50,8 @@ def dump_databases(databases, config, log_prefix, dry_run):
|
||||||
)
|
)
|
||||||
|
|
||||||
dump_path = make_dump_path(config)
|
dump_path = make_dump_path(config)
|
||||||
dump_filename = dump.make_database_dump_filename(dump_path, database['name'])
|
dump_filename = dump.make_data_source_dump_filename(dump_path, database['name'])
|
||||||
|
|
||||||
if os.path.exists(dump_filename):
|
if os.path.exists(dump_filename):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f'{log_prefix}: Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}'
|
f'{log_prefix}: Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}'
|
||||||
|
@ -48,10 +60,10 @@ def dump_databases(databases, config, log_prefix, dry_run):
|
||||||
|
|
||||||
command = (
|
command = (
|
||||||
'sqlite3',
|
'sqlite3',
|
||||||
database_path,
|
shlex.quote(database_path),
|
||||||
'.dump',
|
'.dump',
|
||||||
'>',
|
'>',
|
||||||
dump_filename,
|
shlex.quote(dump_filename),
|
||||||
)
|
)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f'{log_prefix}: Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}'
|
f'{log_prefix}: Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}'
|
||||||
|
@ -59,43 +71,43 @@ def dump_databases(databases, config, log_prefix, dry_run):
|
||||||
if dry_run:
|
if dry_run:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
dump.create_parent_directory_for_dump(dump_filename)
|
dump.create_named_pipe_for_dump(dump_filename)
|
||||||
processes.append(execute_command(command, shell=True, run_to_completion=False))
|
processes.append(execute_command(command, shell=True, run_to_completion=False))
|
||||||
|
|
||||||
return processes
|
return processes
|
||||||
|
|
||||||
|
|
||||||
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Remove the given SQLite3 database dumps from the filesystem. The databases are supplied as a
|
Remove the given SQLite database dumps from the filesystem. The databases are supplied as a
|
||||||
sequence of configuration dicts, as per the configuration schema. Use the given configuration
|
sequence of configuration dicts, as per the configuration schema. Use the given configuration
|
||||||
dict to construct the destination path and the given log prefix in any log entries. If this is a
|
dict to construct the destination path and the given log prefix in any log entries. If this is a
|
||||||
dry run, then don't actually remove anything.
|
dry run, then don't actually remove anything.
|
||||||
'''
|
'''
|
||||||
dump.remove_database_dumps(make_dump_path(config), 'SQLite', log_prefix, dry_run)
|
dump.remove_data_source_dumps(make_dump_path(config), 'SQLite', log_prefix, dry_run)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Make a pattern that matches the given SQLite3 databases. The databases are supplied as a
|
Make a pattern that matches the given SQLite databases. The databases are supplied as a sequence
|
||||||
sequence of configuration dicts, as per the configuration schema.
|
of configuration dicts, as per the configuration schema.
|
||||||
'''
|
'''
|
||||||
return dump.make_database_dump_filename(make_dump_path(config), name)
|
return dump.make_data_source_dump_filename(make_dump_path(config), name)
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dump(
|
def restore_data_source_dump(
|
||||||
hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
|
hook_config, config, log_prefix, data_source, dry_run, extract_process, connection_params
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Restore a database from the given extract stream. The database is supplied as a configuration
|
Restore a database from the given extract stream. The database is supplied as a data source
|
||||||
dict, but the given hook configuration is ignored. The given configuration dict is used to
|
configuration dict, but the given hook configuration is ignored. The given configuration dict is
|
||||||
construct the destination path, and the given log prefix is used for any log entries. If this is
|
used to construct the destination path, and the given log prefix is used for any log entries. If
|
||||||
a dry run, then don't actually restore anything. Trigger the given active extract process (an
|
this is a dry run, then don't actually restore anything. Trigger the given active extract
|
||||||
instance of subprocess.Popen) to produce output to consume.
|
process (an instance of subprocess.Popen) to produce output to consume.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
||||||
database_path = connection_params['restore_path'] or database.get(
|
database_path = connection_params['restore_path'] or data_source.get(
|
||||||
'restore_path', database.get('path')
|
'restore_path', data_source.get('path')
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
|
logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
|
||||||
|
|
|
@ -41,6 +41,9 @@ def should_do_markup(no_color, configs):
|
||||||
if any(config.get('output', {}).get('color') is False for config in configs.values()):
|
if any(config.get('output', {}).get('color') is False for config in configs.values()):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
if os.environ.get('NO_COLOR', None):
|
||||||
|
return False
|
||||||
|
|
||||||
py_colors = os.environ.get('PY_COLORS', None)
|
py_colors = os.environ.get('PY_COLORS', None)
|
||||||
|
|
||||||
if py_colors is not None:
|
if py_colors is not None:
|
||||||
|
@ -159,22 +162,23 @@ def configure_logging(
|
||||||
monitoring_log_level=None,
|
monitoring_log_level=None,
|
||||||
log_file=None,
|
log_file=None,
|
||||||
log_file_format=None,
|
log_file_format=None,
|
||||||
|
color_enabled=True,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Configure logging to go to both the console and (syslog or log file). Use the given log levels,
|
Configure logging to go to both the console and (syslog or log file). Use the given log levels,
|
||||||
respectively.
|
respectively. If color is enabled, set up log formatting accordingly.
|
||||||
|
|
||||||
Raise FileNotFoundError or PermissionError if the log file could not be opened for writing.
|
Raise FileNotFoundError or PermissionError if the log file could not be opened for writing.
|
||||||
'''
|
'''
|
||||||
|
add_custom_log_levels()
|
||||||
|
|
||||||
if syslog_log_level is None:
|
if syslog_log_level is None:
|
||||||
syslog_log_level = console_log_level
|
syslog_log_level = logging.DISABLED
|
||||||
if log_file_log_level is None:
|
if log_file_log_level is None:
|
||||||
log_file_log_level = console_log_level
|
log_file_log_level = console_log_level
|
||||||
if monitoring_log_level is None:
|
if monitoring_log_level is None:
|
||||||
monitoring_log_level = console_log_level
|
monitoring_log_level = console_log_level
|
||||||
|
|
||||||
add_custom_log_levels()
|
|
||||||
|
|
||||||
# Log certain log levels to console stderr and others to stdout. This supports use cases like
|
# Log certain log levels to console stderr and others to stdout. This supports use cases like
|
||||||
# grepping (non-error) output.
|
# grepping (non-error) output.
|
||||||
console_disabled = logging.NullHandler()
|
console_disabled = logging.NullHandler()
|
||||||
|
@ -191,11 +195,17 @@ def configure_logging(
|
||||||
logging.DEBUG: console_standard_handler,
|
logging.DEBUG: console_standard_handler,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
console_handler.setFormatter(Console_color_formatter())
|
|
||||||
|
if color_enabled:
|
||||||
|
console_handler.setFormatter(Console_color_formatter())
|
||||||
|
|
||||||
console_handler.setLevel(console_log_level)
|
console_handler.setLevel(console_log_level)
|
||||||
|
|
||||||
syslog_path = None
|
handlers = [console_handler]
|
||||||
if log_file is None and syslog_log_level != logging.DISABLED:
|
|
||||||
|
if syslog_log_level != logging.DISABLED:
|
||||||
|
syslog_path = None
|
||||||
|
|
||||||
if os.path.exists('/dev/log'):
|
if os.path.exists('/dev/log'):
|
||||||
syslog_path = '/dev/log'
|
syslog_path = '/dev/log'
|
||||||
elif os.path.exists('/var/run/syslog'):
|
elif os.path.exists('/var/run/syslog'):
|
||||||
|
@ -203,14 +213,15 @@ def configure_logging(
|
||||||
elif os.path.exists('/var/run/log'):
|
elif os.path.exists('/var/run/log'):
|
||||||
syslog_path = '/var/run/log'
|
syslog_path = '/var/run/log'
|
||||||
|
|
||||||
if syslog_path and not interactive_console():
|
if syslog_path:
|
||||||
syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
|
syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
|
||||||
syslog_handler.setFormatter(
|
syslog_handler.setFormatter(
|
||||||
logging.Formatter('borgmatic: {levelname} {message}', style='{') # noqa: FS003
|
logging.Formatter('borgmatic: {levelname} {message}', style='{') # noqa: FS003
|
||||||
)
|
)
|
||||||
syslog_handler.setLevel(syslog_log_level)
|
syslog_handler.setLevel(syslog_log_level)
|
||||||
handlers = (console_handler, syslog_handler)
|
handlers.append(syslog_handler)
|
||||||
elif log_file and log_file_log_level != logging.DISABLED:
|
|
||||||
|
if log_file and log_file_log_level != logging.DISABLED:
|
||||||
file_handler = logging.handlers.WatchedFileHandler(log_file)
|
file_handler = logging.handlers.WatchedFileHandler(log_file)
|
||||||
file_handler.setFormatter(
|
file_handler.setFormatter(
|
||||||
logging.Formatter(
|
logging.Formatter(
|
||||||
|
@ -218,11 +229,9 @@ def configure_logging(
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
file_handler.setLevel(log_file_log_level)
|
file_handler.setLevel(log_file_log_level)
|
||||||
handlers = (console_handler, file_handler)
|
handlers.append(file_handler)
|
||||||
else:
|
|
||||||
handlers = (console_handler,)
|
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=min(console_log_level, syslog_log_level, log_file_log_level, monitoring_log_level),
|
level=min(handler.level for handler in handlers),
|
||||||
handlers=handlers,
|
handlers=handlers,
|
||||||
)
|
)
|
||||||
|
|
|
@ -23,12 +23,20 @@ def handle_signal(signal_number, frame):
|
||||||
if signal_number == signal.SIGTERM:
|
if signal_number == signal.SIGTERM:
|
||||||
logger.critical('Exiting due to TERM signal')
|
logger.critical('Exiting due to TERM signal')
|
||||||
sys.exit(EXIT_CODE_FROM_SIGNAL + signal.SIGTERM)
|
sys.exit(EXIT_CODE_FROM_SIGNAL + signal.SIGTERM)
|
||||||
|
elif signal_number == signal.SIGINT:
|
||||||
|
raise KeyboardInterrupt()
|
||||||
|
|
||||||
|
|
||||||
def configure_signals():
|
def configure_signals():
|
||||||
'''
|
'''
|
||||||
Configure borgmatic's signal handlers to pass relevant signals through to any child processes
|
Configure borgmatic's signal handlers to pass relevant signals through to any child processes
|
||||||
like Borg. Note that SIGINT gets passed through even without these changes.
|
like Borg.
|
||||||
'''
|
'''
|
||||||
for signal_number in (signal.SIGHUP, signal.SIGTERM, signal.SIGUSR1, signal.SIGUSR2):
|
for signal_number in (
|
||||||
|
signal.SIGHUP,
|
||||||
|
signal.SIGINT,
|
||||||
|
signal.SIGTERM,
|
||||||
|
signal.SIGUSR1,
|
||||||
|
signal.SIGUSR2,
|
||||||
|
):
|
||||||
signal.signal(signal_number, handle_signal)
|
signal.signal(signal_number, handle_signal)
|
||||||
|
|
|
@ -2,13 +2,13 @@
|
||||||
font-size: 1rem; /* Reset */
|
font-size: 1rem; /* Reset */
|
||||||
}
|
}
|
||||||
.elv-toc details {
|
.elv-toc details {
|
||||||
--details-force-closed: (max-width: 63.9375em); /* 1023px */
|
--details-force-closed: (max-width: 79.9375em); /* 1023px */
|
||||||
}
|
}
|
||||||
.elv-toc details > summary {
|
.elv-toc details > summary {
|
||||||
font-size: 1.375rem; /* 22px /16 */
|
font-size: 1.375rem; /* 22px /16 */
|
||||||
margin-bottom: .5em;
|
margin-bottom: .5em;
|
||||||
}
|
}
|
||||||
@media (min-width: 64em) { /* 1024px */
|
@media (min-width: 80em) {
|
||||||
.elv-toc {
|
.elv-toc {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
left: 3rem;
|
left: 3rem;
|
||||||
|
|
|
@ -121,7 +121,7 @@ main h1:first-child,
|
||||||
main .elv-toc + h1 {
|
main .elv-toc + h1 {
|
||||||
border-bottom: 2px dotted #666;
|
border-bottom: 2px dotted #666;
|
||||||
}
|
}
|
||||||
@media (min-width: 64em) { /* 1024px */
|
@media (min-width: 80em) {
|
||||||
main .elv-toc + h1,
|
main .elv-toc + h1,
|
||||||
main .elv-toc + h2 {
|
main .elv-toc + h2 {
|
||||||
margin-top: 0;
|
margin-top: 0;
|
||||||
|
@ -243,10 +243,10 @@ footer.elv-layout {
|
||||||
.elv-layout-full {
|
.elv-layout-full {
|
||||||
max-width: none;
|
max-width: none;
|
||||||
}
|
}
|
||||||
@media (min-width: 64em) { /* 1024px */
|
@media (min-width: 80em) {
|
||||||
.elv-layout-toc {
|
.elv-layout-toc {
|
||||||
padding-left: 15rem;
|
padding-left: 15rem;
|
||||||
max-width: 60rem;
|
max-width: 76rem;
|
||||||
margin-right: 1rem;
|
margin-right: 1rem;
|
||||||
position: relative;
|
position: relative;
|
||||||
}
|
}
|
||||||
|
|
|
@ -126,7 +126,7 @@ for more information.
|
||||||
## Hook output
|
## Hook output
|
||||||
|
|
||||||
Any output produced by your hooks shows up both at the console and in syslog
|
Any output produced by your hooks shows up both at the console and in syslog
|
||||||
(when run in a non-interactive console). For more information, read about <a
|
(when enabled). For more information, read about <a
|
||||||
href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/">inspecting
|
href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/">inspecting
|
||||||
your backups</a>.
|
your backups</a>.
|
||||||
|
|
||||||
|
|
|
@ -206,6 +206,36 @@ hooks:
|
||||||
|
|
||||||
Alter the ports in these examples to suit your particular database system.
|
Alter the ports in these examples to suit your particular database system.
|
||||||
|
|
||||||
|
Normally, borgmatic dumps a database by running a database dump command (e.g.
|
||||||
|
`pg_dump`) on the host or wherever borgmatic is running, and this command
|
||||||
|
connects to your containerized database via the given `hostname` and `port`.
|
||||||
|
But if you don't have any database dump commands installed on your host and
|
||||||
|
you'd rather use the commands inside your database container itself, borgmatic
|
||||||
|
supports that too. Just configure borgmatic to `exec` into your container to
|
||||||
|
run the dump command.
|
||||||
|
|
||||||
|
For instance, if using Docker and PostgreSQL, something like this might work:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
hooks:
|
||||||
|
postgresql_databases:
|
||||||
|
- name: users
|
||||||
|
hostname: 127.0.0.1
|
||||||
|
port: 5433
|
||||||
|
username: postgres
|
||||||
|
password: trustsome1
|
||||||
|
pg_dump_command: docker exec my_pg_container pg_dump
|
||||||
|
```
|
||||||
|
|
||||||
|
... where `my_pg_container` is the name of your database container. In this
|
||||||
|
example, you'd also need to set the `pg_restore_command` and `psql_command`
|
||||||
|
options.
|
||||||
|
|
||||||
|
Similar command override options are available for (some of) the other
|
||||||
|
supported database types as well. See the [configuration
|
||||||
|
reference](https://torsion.org/borgmatic/docs/reference/configuration/) for
|
||||||
|
details.
|
||||||
|
|
||||||
|
|
||||||
### No source directories
|
### No source directories
|
||||||
|
|
||||||
|
@ -262,6 +292,10 @@ systems that you'd like supported.
|
||||||
|
|
||||||
## Database restoration
|
## Database restoration
|
||||||
|
|
||||||
|
When you want to replace an existing database with its backed-up contents, you
|
||||||
|
can restore it with borgmatic. Note that the database must already exist;
|
||||||
|
borgmatic does not currently create a database upon restore.
|
||||||
|
|
||||||
To restore a database dump from an archive, use the `borgmatic restore`
|
To restore a database dump from an archive, use the `borgmatic restore`
|
||||||
action. But the first step is to figure out which archive to restore from. A
|
action. But the first step is to figure out which archive to restore from. A
|
||||||
good way to do that is to use the `rlist` action:
|
good way to do that is to use the `rlist` action:
|
||||||
|
@ -403,19 +437,28 @@ borgmatic's own configuration file. So include your configuration file in
|
||||||
backups to avoid getting caught without a way to restore a database.
|
backups to avoid getting caught without a way to restore a database.
|
||||||
3. borgmatic does not currently support backing up or restoring multiple
|
3. borgmatic does not currently support backing up or restoring multiple
|
||||||
databases that share the exact same name on different hosts.
|
databases that share the exact same name on different hosts.
|
||||||
4. Because database hooks implicitly enable the `read_special` configuration,
|
4. When database hooks are enabled, borgmatic instructs Borg to consume
|
||||||
any special files are excluded from backups (named pipes, block devices,
|
special files (via `--read-special`) to support database dump
|
||||||
character devices, and sockets) to prevent hanging. Try a command like `find
|
streaming—regardless of the value of your `read_special` configuration option.
|
||||||
/your/source/path -type b -or -type c -or -type p -or -type s` to find such
|
And because this can cause Borg to hang, borgmatic also automatically excludes
|
||||||
files. Common directories to exclude are `/dev` and `/run`, but that may not
|
special files (and symlinks to them) that Borg may get stuck on. Even so,
|
||||||
be exhaustive. <span class="minilink minilink-addedin">New in version
|
there are still potential edge cases in which applications on your system
|
||||||
1.7.3</span> When database hooks are enabled, borgmatic automatically excludes
|
create new special files *after* borgmatic constructs its exclude list,
|
||||||
special files (and symlinks to special files) that may cause Borg to hang, so
|
resulting in Borg hangs. If that occurs, you can resort to manually excluding
|
||||||
generally you no longer need to manually exclude them. There are potential
|
those files. And if you explicitly set the `read-special` option to `true`,
|
||||||
edge cases though in which applications on your system create new special files
|
borgmatic will opt you out of the auto-exclude feature entirely, but will
|
||||||
*after* borgmatic constructs its exclude list, resulting in Borg hangs. If that
|
still instruct Borg to consume special files—you will just be on your own to
|
||||||
occurs, you can resort to the manual excludes described above. And to opt out
|
exclude them. <span class="minilink minilink-addedin">Prior to version
|
||||||
of the auto-exclude feature entirely, explicitly set `read_special` to true.
|
1.7.3</span>Special files were not auto-excluded, and you were responsible for
|
||||||
|
excluding them yourself. Common directories to exclude are `/dev` and `/run`,
|
||||||
|
but that may not be exhaustive.
|
||||||
|
5. Database hooks also implicitly enable the `one_file_system` option, which
|
||||||
|
means Borg won't cross filesystem boundaries when looking for files to backup.
|
||||||
|
This is especially important when running borgmatic in a container, as
|
||||||
|
container volumes are mounted as separate filesystems. One work-around is to
|
||||||
|
explicitly add each mounted volume you'd like to backup to
|
||||||
|
`source_directories` instead of relying on Borg to include them implicitly via
|
||||||
|
a parent directory.
|
||||||
|
|
||||||
|
|
||||||
### Manual restoration
|
### Manual restoration
|
||||||
|
|
|
@ -0,0 +1,86 @@
|
||||||
|
---
|
||||||
|
title: How to customize warnings and errors
|
||||||
|
eleventyNavigation:
|
||||||
|
key: đź’Ą Customize warnings/errors
|
||||||
|
parent: How-to guides
|
||||||
|
order: 12
|
||||||
|
---
|
||||||
|
## When things go wrong
|
||||||
|
|
||||||
|
After Borg runs, it indicates whether it succeeded via its exit code, a
|
||||||
|
numeric ID indicating success, warning, or error. borgmatic consumes this exit
|
||||||
|
code to decide how to respond. Normally, a Borg error results in a borgmatic
|
||||||
|
error, while a Borg warning or success doesn't.
|
||||||
|
|
||||||
|
But if that default behavior isn't sufficient for your needs, you can
|
||||||
|
customize how borgmatic interprets [Borg's exit
|
||||||
|
codes](https://borgbackup.readthedocs.io/en/stable/usage/general.html#return-codes).
|
||||||
|
For instance, to elevate Borg warnings to errors, thereby causing borgmatic to
|
||||||
|
error on them, use the following borgmatic configuration:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
borg_exit_codes:
|
||||||
|
- exit_code: 1
|
||||||
|
treat_as: error
|
||||||
|
```
|
||||||
|
|
||||||
|
Be aware though that Borg exits with a warning code for a variety of benign
|
||||||
|
situations such as files changing while they're being read, so this example
|
||||||
|
may not meet your needs. Keep reading though for more granular exit code
|
||||||
|
configuration.
|
||||||
|
|
||||||
|
Here's an example that squashes Borg errors to warnings:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
borg_exit_codes:
|
||||||
|
- exit_code: 2
|
||||||
|
treat_as: warning
|
||||||
|
```
|
||||||
|
|
||||||
|
Be careful with this example though, because it prevents borgmatic from
|
||||||
|
erroring when Borg errors, which may not be desirable.
|
||||||
|
|
||||||
|
|
||||||
|
### More granular configuration
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in Borg version 1.4</span> Borg
|
||||||
|
support for [more granular exit
|
||||||
|
codes](https://borgbackup.readthedocs.io/en/1.4-maint/usage/general.html#return-codes)
|
||||||
|
means that you can configure borgmatic to respond to specific Borg conditions.
|
||||||
|
See the full list of [Borg 1.4 error and warning exit
|
||||||
|
codes](https://borgbackup.readthedocs.io/en/1.4.0b1/internals/frontends.html#message-ids).
|
||||||
|
The `rc:` numeric value there tells you the exit code for each.
|
||||||
|
|
||||||
|
For instance, this borgmatic configuration elevates all Borg backup file
|
||||||
|
permission warnings (exit code `105`)—and only those warnings—to errors:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
borg_exit_codes:
|
||||||
|
- exit_code: 105
|
||||||
|
treat_as: error
|
||||||
|
```
|
||||||
|
|
||||||
|
The following configuration does that *and* elevates backup file not found
|
||||||
|
warnings (exit code `107`) to errors as well:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
borg_exit_codes:
|
||||||
|
- exit_code: 105
|
||||||
|
treat_as: error
|
||||||
|
- exit_code: 107
|
||||||
|
treat_as: error
|
||||||
|
```
|
||||||
|
|
||||||
|
If you don't know the exit code for a particular Borg error or warning you're
|
||||||
|
experiencing, you can usually find it in your borgmatic output when
|
||||||
|
`--verbosity 2` is enabled. For instance, here's a snippet of that output when
|
||||||
|
a backup file is not found:
|
||||||
|
|
||||||
|
```
|
||||||
|
/noexist: stat: [Errno 2] No such file or directory: '/noexist'
|
||||||
|
...
|
||||||
|
terminating with warning status, rc 107
|
||||||
|
```
|
||||||
|
|
||||||
|
So if you want to configure borgmatic to treat this as an error instead of a
|
||||||
|
warning, the exit status to use is `107`.
|
|
@ -51,6 +51,11 @@ cron job), while only running expensive consistency checks with `check` on a
|
||||||
much less frequent basis (e.g. with `borgmatic check` called from a separate
|
much less frequent basis (e.g. with `borgmatic check` called from a separate
|
||||||
cron job).
|
cron job).
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.5</span> Instead of
|
||||||
|
(or in addition to) specifying actions on the command-line, you can configure
|
||||||
|
borgmatic to [skip particular
|
||||||
|
actions](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#skipping-actions).
|
||||||
|
|
||||||
|
|
||||||
### Consistency check configuration
|
### Consistency check configuration
|
||||||
|
|
||||||
|
@ -86,8 +91,9 @@ Here are the available checks from fastest to slowest:
|
||||||
|
|
||||||
* `repository`: Checks the consistency of the repository itself.
|
* `repository`: Checks the consistency of the repository itself.
|
||||||
* `archives`: Checks all of the archives in the repository.
|
* `archives`: Checks all of the archives in the repository.
|
||||||
* `extract`: Performs an extraction dry-run of the most recent archive.
|
* `extract`: Performs an extraction dry-run of the latest archive.
|
||||||
* `data`: Verifies the data integrity of all archives contents, decrypting and decompressing all data.
|
* `data`: Verifies the data integrity of all archives contents, decrypting and decompressing all data.
|
||||||
|
* `spot`: Compares file counts and contents between your source files and the latest archive.
|
||||||
|
|
||||||
Note that the `data` check is a more thorough version of the `archives` check,
|
Note that the `data` check is a more thorough version of the `archives` check,
|
||||||
so enabling the `data` check implicitly enables the `archives` check as well.
|
so enabling the `data` check implicitly enables the `archives` check as well.
|
||||||
|
@ -97,6 +103,88 @@ documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html)
|
||||||
for more information.
|
for more information.
|
||||||
|
|
||||||
|
|
||||||
|
### Spot check
|
||||||
|
|
||||||
|
The various consistency checks all have trade-offs around speed and
|
||||||
|
thoroughness, but most of them don't even look at your original source
|
||||||
|
files—arguably one important way to ensure your backups contain the files
|
||||||
|
you'll want to restore in the case of catastrophe (or just an accidentally
|
||||||
|
deleted file). Because if something goes wrong with your source files, most
|
||||||
|
consistency checks will still pass with flying colors and you won't discover
|
||||||
|
there's a problem until you go to restore.
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.10</span> <span
|
||||||
|
class="minilink minilink-addedin">Beta feature</span> That's where the spot
|
||||||
|
check comes in. This check actually compares your source file counts and data
|
||||||
|
against those in the latest archive, potentially catching problems like
|
||||||
|
incorrect excludes, inadvertent deletes, files changed by malware, etc.
|
||||||
|
|
||||||
|
However, because an exhaustive comparison of all source files against the
|
||||||
|
latest archive might be too slow, the spot check supports *sampling* a
|
||||||
|
percentage of your source files for the comparison, ensuring they fall within
|
||||||
|
configured tolerances.
|
||||||
|
|
||||||
|
Here's how it works. Start by installing the `xxhash` OS package if you don't
|
||||||
|
already have it, so the spot check can run the `xxh64sum` command and
|
||||||
|
efficiently hash files for comparison. Then add something like the following
|
||||||
|
to your borgmatic configuration:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
checks:
|
||||||
|
- name: spot
|
||||||
|
count_tolerance_percentage: 10
|
||||||
|
data_sample_percentage: 1
|
||||||
|
data_tolerance_percentage: 0.5
|
||||||
|
```
|
||||||
|
|
||||||
|
The `count_tolerance_percentage` is the percentage delta between the source
|
||||||
|
directories file count and the latest backup archive file count that is
|
||||||
|
allowed before the entire consistency check fails. For instance, if the spot
|
||||||
|
check runs and finds 100 source files on disk and 105 files in the latest
|
||||||
|
archive, that would be within the configured 10% count tolerance and the check
|
||||||
|
would succeed. But if there were 100 source files and 200 archive files, the
|
||||||
|
check would fail. (100 source files and only 50 archive files would also
|
||||||
|
fail.)
|
||||||
|
|
||||||
|
The `data_sample_percentage` is the percentage of total files in the source
|
||||||
|
directories to randomly sample and compare to their corresponding files in the
|
||||||
|
latest backup archive. A higher value allows a more accurate check—and a
|
||||||
|
slower one. The comparison is performed by hashing the selected source files
|
||||||
|
and counting hashes that don't match the latest archive. For instance, if you
|
||||||
|
have 1,000 source files and your sample percentage is 1%, then only 10 source
|
||||||
|
files will be compared against the latest archive. These sampled files are
|
||||||
|
selected randomly each time, so in effect the spot check is probabilistic.
|
||||||
|
|
||||||
|
The `data_tolerance_percentage` is the percentage of total files in the source
|
||||||
|
directories that can fail a spot check data comparison without failing the
|
||||||
|
entire consistency check. The value must be lower than or equal to the
|
||||||
|
`contents_sample_percentage`.
|
||||||
|
|
||||||
|
All three options are required when using the spot check. And because the
|
||||||
|
check relies on these configured tolerances, it may not be a
|
||||||
|
set-it-and-forget-it type of consistency check, at least until you get the
|
||||||
|
tolerances dialed in so there are minimal false positives or negatives. It is
|
||||||
|
recommended you run `borgmatic check` several times after configuring the spot
|
||||||
|
check, tweaking your tolerances as needed. For certain workloads where your
|
||||||
|
source files experience wild swings of file contents or counts, the spot check
|
||||||
|
may not suitable at all.
|
||||||
|
|
||||||
|
What if you add, delete, or change a bunch of your source files and you don't
|
||||||
|
want the spot check to fail the next time it's run? Run `borgmatic create` to
|
||||||
|
create a new backup, thereby allowing the next spot check to run against an
|
||||||
|
archive that contains your recent changes.
|
||||||
|
|
||||||
|
Because the spot check only looks at the most recent archive, you may not want
|
||||||
|
to run it immediately after a `create` action (borgmatic's default behavior).
|
||||||
|
Instead, it may make more sense to run the spot check on a separate schedule
|
||||||
|
from `create`.
|
||||||
|
|
||||||
|
As long as the spot check feature is in beta, it may be subject to breaking
|
||||||
|
changes. But feel free to use it in production if you're okay with that
|
||||||
|
caveat, and please [provide any
|
||||||
|
feedback](https://torsion.org/borgmatic/#issues) you have on this feature.
|
||||||
|
|
||||||
|
|
||||||
### Check frequency
|
### Check frequency
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">New in version 1.6.2</span> You can
|
<span class="minilink minilink-addedin">New in version 1.6.2</span> You can
|
||||||
|
@ -116,8 +204,17 @@ this option in the `consistency:` section of your configuration.
|
||||||
|
|
||||||
This tells borgmatic to run the `repository` consistency check at most once
|
This tells borgmatic to run the `repository` consistency check at most once
|
||||||
every two weeks for a given repository and the `archives` check at most once a
|
every two weeks for a given repository and the `archives` check at most once a
|
||||||
month. The `frequency` value is a number followed by a unit of time, e.g. "3
|
month. The `frequency` value is a number followed by a unit of time, e.g. `3
|
||||||
days", "1 week", "2 months", etc.
|
days`, `1 week`, `2 months`, etc. The set of possible time units is as
|
||||||
|
follows (singular or plural):
|
||||||
|
|
||||||
|
* `second`
|
||||||
|
* `minute`
|
||||||
|
* `hour`
|
||||||
|
* `day`
|
||||||
|
* `week` (7 days)
|
||||||
|
* `month` (30 days)
|
||||||
|
* `year` (365 days)
|
||||||
|
|
||||||
The `frequency` defaults to `always` for a check configured without a
|
The `frequency` defaults to `always` for a check configured without a
|
||||||
`frequency`, which means run this check every time checks run. But if you omit
|
`frequency`, which means run this check every time checks run. But if you omit
|
||||||
|
@ -139,6 +236,10 @@ though—or the most frequently configured check will apply.
|
||||||
If you want to temporarily ignore your configured frequencies, you can invoke
|
If you want to temporarily ignore your configured frequencies, you can invoke
|
||||||
`borgmatic check --force` to run checks unconditionally.
|
`borgmatic check --force` to run checks unconditionally.
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.6</span> `borgmatic
|
||||||
|
check --force` runs `check` even if it's specified in the `skip_actions`
|
||||||
|
option.
|
||||||
|
|
||||||
|
|
||||||
### Running only checks
|
### Running only checks
|
||||||
|
|
||||||
|
@ -162,7 +263,16 @@ location:
|
||||||
If that's still too slow, you can disable consistency checks entirely,
|
If that's still too slow, you can disable consistency checks entirely,
|
||||||
either for a single repository or for all repositories.
|
either for a single repository or for all repositories.
|
||||||
|
|
||||||
Disabling all consistency checks looks like this:
|
<span class="minilink minilink-addedin">New in version 1.8.5</span> Disabling
|
||||||
|
all consistency checks looks like this:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
skip_actions:
|
||||||
|
- check
|
||||||
|
```
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">Prior to version 1.8.5</span> Use this
|
||||||
|
configuration instead:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
checks:
|
checks:
|
||||||
|
@ -170,10 +280,10 @@ checks:
|
||||||
```
|
```
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
||||||
this option in the `consistency:` section of your configuration.
|
`checks:` in the `consistency:` section of your configuration.
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">Prior to version 1.6.2</span> `checks`
|
<span class="minilink minilink-addedin">Prior to version 1.6.2</span>
|
||||||
was a plain list of strings without the `name:` part. For instance:
|
`checks:` was a plain list of strings without the `name:` part. For instance:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
checks:
|
checks:
|
||||||
|
|
|
@ -3,11 +3,16 @@ title: How to develop on borgmatic
|
||||||
eleventyNavigation:
|
eleventyNavigation:
|
||||||
key: 🏗️ Develop on borgmatic
|
key: 🏗️ Develop on borgmatic
|
||||||
parent: How-to guides
|
parent: How-to guides
|
||||||
order: 13
|
order: 14
|
||||||
---
|
---
|
||||||
## Source code
|
## Source code
|
||||||
|
|
||||||
To get set up to develop on borgmatic, first clone it via HTTPS or SSH:
|
To get set up to develop on borgmatic, first [`install
|
||||||
|
pipx`](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#installation)
|
||||||
|
to make managing your borgmatic environment easy without impacting other
|
||||||
|
Python applications on your system.
|
||||||
|
|
||||||
|
Then, clone borgmatic via HTTPS or SSH:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://projects.torsion.org/borgmatic-collective/borgmatic.git
|
git clone https://projects.torsion.org/borgmatic-collective/borgmatic.git
|
||||||
|
@ -19,23 +24,24 @@ Or:
|
||||||
git clone ssh://git@projects.torsion.org:3022/borgmatic-collective/borgmatic.git
|
git clone ssh://git@projects.torsion.org:3022/borgmatic-collective/borgmatic.git
|
||||||
```
|
```
|
||||||
|
|
||||||
Then, install borgmatic
|
Finally, install borgmatic
|
||||||
"[editable](https://pip.pypa.io/en/stable/cli/pip_install/#editable-installs)"
|
"[editable](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs)"
|
||||||
so that you can run borgmatic actions during development to make sure your
|
so that you can run borgmatic actions during development to make sure your
|
||||||
changes work.
|
changes work:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd borgmatic
|
cd borgmatic
|
||||||
pip3 install --user --editable .
|
pipx ensurepath
|
||||||
|
pipx install --editable .
|
||||||
```
|
```
|
||||||
|
|
||||||
Note that this will typically install the borgmatic commands into
|
Or to work on the [Apprise
|
||||||
`~/.local/bin`, which may or may not be on your PATH. There are other ways to
|
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook),
|
||||||
install borgmatic editable as well, for instance into the system Python
|
change that last line to:
|
||||||
install (so without `--user`, as root), or even into a
|
|
||||||
[virtualenv](https://virtualenv.pypa.io/en/stable/). How or where you install
|
```bash
|
||||||
borgmatic is up to you, but generally an editable install makes development
|
pipx install --editable .[Apprise]
|
||||||
and testing easier.
|
```
|
||||||
|
|
||||||
To get oriented with the borgmatic source code, have a look at the [source
|
To get oriented with the borgmatic source code, have a look at the [source
|
||||||
code reference](https://torsion.org/borgmatic/docs/reference/source-code/).
|
code reference](https://torsion.org/borgmatic/docs/reference/source-code/).
|
||||||
|
@ -43,15 +49,17 @@ code reference](https://torsion.org/borgmatic/docs/reference/source-code/).
|
||||||
|
|
||||||
## Automated tests
|
## Automated tests
|
||||||
|
|
||||||
Assuming you've cloned the borgmatic source code as described above, and
|
Assuming you've cloned the borgmatic source code as described above and you're
|
||||||
you're in the `borgmatic/` working copy, install tox, which is used for
|
in the `borgmatic/` working copy, install tox, which is used for setting up
|
||||||
setting up testing environments:
|
testing environments. You can either install a system package of tox (likely
|
||||||
|
called `tox` or `python-tox`) or you can install tox with pipx:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip3 install --user tox
|
pipx install tox
|
||||||
```
|
```
|
||||||
|
|
||||||
Finally, to actually run tests, run:
|
Finally, to actually run tests, run tox from inside the borgmatic
|
||||||
|
sourcedirectory:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
tox
|
tox
|
||||||
|
@ -92,14 +100,14 @@ with Borg and supported databases for a few representative scenarios. These
|
||||||
tests don't run by default when running `tox`, because they're relatively slow
|
tests don't run by default when running `tox`, because they're relatively slow
|
||||||
and depend on containers for runtime dependencies. These tests do run on the
|
and depend on containers for runtime dependencies. These tests do run on the
|
||||||
continuous integration (CI) server, and running them on your developer machine
|
continuous integration (CI) server, and running them on your developer machine
|
||||||
is the closest thing to CI-test parity.
|
is the closest thing to dev-CI parity.
|
||||||
|
|
||||||
If you would like to run the full test suite, first install Docker (or Podman;
|
If you would like to run the full test suite, first install Docker (or Podman;
|
||||||
see below) and [Docker Compose](https://docs.docker.com/compose/install/).
|
see below) and [Docker Compose](https://docs.docker.com/compose/install/).
|
||||||
Then run:
|
Then run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
scripts/run-end-to-end-dev-tests
|
scripts/run-end-to-end-tests
|
||||||
```
|
```
|
||||||
|
|
||||||
This script assumes you have permission to run `docker`. If you don't, then
|
This script assumes you have permission to run `docker`. If you don't, then
|
||||||
|
@ -141,6 +149,9 @@ the following deviations from it:
|
||||||
separate from their contents.
|
separate from their contents.
|
||||||
* Within multiline constructs, use standard four-space indentation. Don't align
|
* Within multiline constructs, use standard four-space indentation. Don't align
|
||||||
indentation with an opening delimiter.
|
indentation with an opening delimiter.
|
||||||
|
* In general, spell out words in variable names instead of shortening them.
|
||||||
|
So, think `index` instead of `idx`. There are some notable exceptions to
|
||||||
|
this though (like `config`).
|
||||||
|
|
||||||
borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
|
borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
|
||||||
formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and
|
formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and
|
||||||
|
@ -148,12 +159,17 @@ the [isort](https://github.com/timothycrosley/isort) import orderer, so
|
||||||
certain code style requirements will be enforced when running automated tests.
|
certain code style requirements will be enforced when running automated tests.
|
||||||
See the Black, Flake8, and isort documentation for more information.
|
See the Black, Flake8, and isort documentation for more information.
|
||||||
|
|
||||||
|
|
||||||
## Continuous integration
|
## Continuous integration
|
||||||
|
|
||||||
Each pull request triggers a continuous integration build which runs the test
|
Each commit to
|
||||||
suite. You can view these builds on
|
[main](https://projects.torsion.org/borgmatic-collective/borgmatic/branches)
|
||||||
[build.torsion.org](https://build.torsion.org/borgmatic-collective/borgmatic),
|
triggers [a continuous integration
|
||||||
and they're also linked from the commits list on each pull request.
|
build](https://projects.torsion.org/borgmatic-collective/borgmatic/actions)
|
||||||
|
which runs the test suite and updates
|
||||||
|
[documentation](https://torsion.org/borgmatic/). These builds are also linked
|
||||||
|
from the [commits for the main
|
||||||
|
branch](https://projects.torsion.org/borgmatic-collective/borgmatic/commits/branch/main).
|
||||||
|
|
||||||
## Documentation development
|
## Documentation development
|
||||||
|
|
||||||
|
|
|
@ -149,9 +149,10 @@ borgmatic umount --mount-point /mnt
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">New in version 1.7.15</span> borgmatic
|
<span class="minilink minilink-addedin">New in version 1.7.15</span> borgmatic
|
||||||
automatically stores all the configuration files used to create an archive
|
automatically stores all the configuration files used to create an archive
|
||||||
inside the archive itself. This is useful in cases where you've lost a
|
inside the archive itself. They are stored in the archive using their full
|
||||||
configuration file or you want to see what configurations were used to create a
|
paths from the machine being backed up. This is useful in cases where you've
|
||||||
particular archive.
|
lost a configuration file or you want to see what configurations were used to
|
||||||
|
create a particular archive.
|
||||||
|
|
||||||
To extract the configuration files from an archive, use the `config bootstrap`
|
To extract the configuration files from an archive, use the `config bootstrap`
|
||||||
action. For example:
|
action. For example:
|
||||||
|
@ -166,8 +167,8 @@ configuration file used to create this archive was located at
|
||||||
`/etc/borgmatic/config.yaml` when the archive was created.
|
`/etc/borgmatic/config.yaml` when the archive was created.
|
||||||
|
|
||||||
Note that to run the `config bootstrap` action, you don't need to have a
|
Note that to run the `config bootstrap` action, you don't need to have a
|
||||||
borgmatic configuration file. You only need to specify the repository to use via
|
borgmatic configuration file. You only need to specify the repository to use
|
||||||
the `--repository` flag; borgmatic will figure out the rest.
|
via the `--repository` flag; borgmatic will figure out the rest.
|
||||||
|
|
||||||
If a destination directory is not specified, the configuration files will be
|
If a destination directory is not specified, the configuration files will be
|
||||||
extracted to their original locations, silently *overwriting* any configuration
|
extracted to their original locations, silently *overwriting* any configuration
|
||||||
|
@ -182,8 +183,16 @@ If you want to extract the configuration file from a specific archive, use the
|
||||||
borgmatic config bootstrap --repository repo.borg --archive host-2023-01-02T04:06:07.080910 --destination /tmp
|
borgmatic config bootstrap --repository repo.borg --archive host-2023-01-02T04:06:07.080910 --destination /tmp
|
||||||
```
|
```
|
||||||
|
|
||||||
|
See the output of `config bootstrap --help` for additional flags you may need
|
||||||
|
for bootstrapping.
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">New in version 1.8.1</span> Set the
|
<span class="minilink minilink-addedin">New in version 1.8.1</span> Set the
|
||||||
`store_config_files` option to `false` to disable the automatic backup of
|
`store_config_files` option to `false` to disable the automatic backup of
|
||||||
borgmatic configuration files, for instance if they contain sensitive
|
borgmatic configuration files, for instance if they contain sensitive
|
||||||
information you don't want to store even inside your encrypted backups. If you
|
information you don't want to store even inside your encrypted backups. If you
|
||||||
do this though, the `config bootstrap` action will no longer work.
|
do this though, the `config bootstrap` action will no longer work.
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.7</span> Included
|
||||||
|
configuration files are stored in each backup archive. This means that the
|
||||||
|
`config bootstrap` action not only extracts the top-level configuration files
|
||||||
|
but also the includes they depend upon.
|
||||||
|
|
|
@ -116,27 +116,30 @@ archive, complete with file sizes.
|
||||||
|
|
||||||
## Logging
|
## Logging
|
||||||
|
|
||||||
By default, borgmatic logs to a local syslog-compatible daemon if one is
|
By default, borgmatic logs to the console. You can enable simultaneous syslog
|
||||||
present and borgmatic is running in a non-interactive console. Where those
|
logging and customize its log level with the `--syslog-verbosity` flag, which
|
||||||
logs show up depends on your particular system. If you're using systemd, try
|
is independent from the console logging `--verbosity` flag described above.
|
||||||
running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or
|
For instance, to enable syslog logging, run:
|
||||||
similar.
|
|
||||||
|
|
||||||
You can customize the log level used for syslog logging with the
|
|
||||||
`--syslog-verbosity` flag, and this is independent from the console logging
|
|
||||||
`--verbosity` flag described above. For instance, to get additional
|
|
||||||
information about the progress of the backup as it proceeds:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
borgmatic --syslog-verbosity 1
|
borgmatic --syslog-verbosity 1
|
||||||
```
|
```
|
||||||
|
|
||||||
Or to increase syslog logging to include debug spew:
|
To increase syslog logging further to include debugging information, run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
borgmatic --syslog-verbosity 2
|
borgmatic --syslog-verbosity 2
|
||||||
```
|
```
|
||||||
|
|
||||||
|
See above for further details about the verbosity levels.
|
||||||
|
|
||||||
|
Where these logs show up depends on your particular system. If you're using
|
||||||
|
systemd, try running `journalctl -xe`. Otherwise, try viewing
|
||||||
|
`/var/log/syslog` or similar.
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">Prior to version 1.8.3</span>borgmatic
|
||||||
|
logged to syslog by default whenever run at a non-interactive console.
|
||||||
|
|
||||||
### Rate limiting
|
### Rate limiting
|
||||||
|
|
||||||
If you are using rsyslog or systemd's journal, be aware that by default they
|
If you are using rsyslog or systemd's journal, be aware that by default they
|
||||||
|
@ -165,7 +168,7 @@ Note that if you use the `--log-file` flag, you are responsible for rotating
|
||||||
the log file so it doesn't grow too large, for example with
|
the log file so it doesn't grow too large, for example with
|
||||||
[logrotate](https://wiki.archlinux.org/index.php/Logrotate).
|
[logrotate](https://wiki.archlinux.org/index.php/Logrotate).
|
||||||
|
|
||||||
You can the `--log-file-verbosity` flag to customize the log file's log level:
|
You can use the `--log-file-verbosity` flag to customize the log file's log level:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
borgmatic --log-file /path/to/file.log --log-file-verbosity 2
|
borgmatic --log-file /path/to/file.log --log-file-verbosity 2
|
||||||
|
@ -197,5 +200,5 @@ See the [Python logging
|
||||||
documentation](https://docs.python.org/3/library/logging.html#logrecord-attributes)
|
documentation](https://docs.python.org/3/library/logging.html#logrecord-attributes)
|
||||||
for additional placeholders.
|
for additional placeholders.
|
||||||
|
|
||||||
Note that this `--log-file-format` flg only applies to the specified
|
Note that this `--log-file-format` flag only applies to the specified
|
||||||
`--log-file` and not to syslog or other logging.
|
`--log-file` and not to syslog or other logging.
|
||||||
|
|
|
@ -139,8 +139,8 @@ Some borgmatic command-line actions also have a `--match-archives` flag that
|
||||||
overrides both the auto-matching behavior and the `match_archives`
|
overrides both the auto-matching behavior and the `match_archives`
|
||||||
configuration option.
|
configuration option.
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">Prior to 1.7.11</span> The way to
|
<span class="minilink minilink-addedin">Prior to version 1.7.11</span> The way
|
||||||
limit the archives used for the `prune` action was a `prefix` option in the
|
to limit the archives used for the `prune` action was a `prefix` option in the
|
||||||
`retention` section for matching against the start of archive names. And the
|
`retention` section for matching against the start of archive names. And the
|
||||||
option for limiting the archives used for the `check` action was a separate
|
option for limiting the archives used for the `check` action was a separate
|
||||||
`prefix` in the `consistency` section. Both of these options are deprecated in
|
`prefix` in the `consistency` section. Both of these options are deprecated in
|
||||||
|
@ -151,7 +151,7 @@ in newer versions of borgmatic.
|
||||||
## Configuration includes
|
## Configuration includes
|
||||||
|
|
||||||
Once you have multiple different configuration files, you might want to share
|
Once you have multiple different configuration files, you might want to share
|
||||||
common configuration options across these files with having to copy and paste
|
common configuration options across these files without having to copy and paste
|
||||||
them. To achieve this, you can put fragments of common configuration options
|
them. To achieve this, you can put fragments of common configuration options
|
||||||
into a file and then include or inline that file into one or more borgmatic
|
into a file and then include or inline that file into one or more borgmatic
|
||||||
configuration files.
|
configuration files.
|
||||||
|
@ -301,7 +301,7 @@ options via an include and then overrides one of them locally:
|
||||||
<<: !include /etc/borgmatic/common.yaml
|
<<: !include /etc/borgmatic/common.yaml
|
||||||
|
|
||||||
constants:
|
constants:
|
||||||
hostname: myhostname
|
base_directory: /opt
|
||||||
|
|
||||||
repositories:
|
repositories:
|
||||||
- path: repo.borg
|
- path: repo.borg
|
||||||
|
@ -311,13 +311,13 @@ This is what `common.yaml` might look like:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
constants:
|
constants:
|
||||||
prefix: myprefix
|
app_name: myapp
|
||||||
hostname: otherhost
|
base_directory: /var/lib
|
||||||
```
|
```
|
||||||
|
|
||||||
Once this include gets merged in, the resulting configuration would have a
|
Once this include gets merged in, the resulting configuration would have an
|
||||||
`prefix` value of `myprefix` and an overridden `hostname` value of
|
`app_name` value of `myapp` and an overridden `base_directory` value of
|
||||||
`myhostname`.
|
`/opt`.
|
||||||
|
|
||||||
When there's an option collision between the local file and the merged
|
When there's an option collision between the local file and the merged
|
||||||
include, the local file's option takes precedence.
|
include, the local file's option takes precedence.
|
||||||
|
@ -495,21 +495,29 @@ borgmatic create --override parent_option.option1=value1 --override parent_optio
|
||||||
forget to specify the section that an option is in. That looks like a prefix
|
forget to specify the section that an option is in. That looks like a prefix
|
||||||
on the option name, e.g. `location.repositories`.
|
on the option name, e.g. `location.repositories`.
|
||||||
|
|
||||||
Note that each value is parsed as an actual YAML string, so you can even set
|
Note that each value is parsed as an actual YAML string, so you can set list
|
||||||
list values by using brackets. For instance:
|
values by using brackets. For instance:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
borgmatic create --override repositories=[test1.borg,test2.borg]
|
borgmatic create --override repositories=[test1.borg,test2.borg]
|
||||||
```
|
```
|
||||||
|
|
||||||
Or even a single list element:
|
Or a single list element:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
borgmatic create --override repositories=[/root/test.borg]
|
borgmatic create --override repositories=[/root/test.borg]
|
||||||
```
|
```
|
||||||
|
|
||||||
If your override value contains special YAML characters like colons, then
|
Or a single list element that is a key/value pair:
|
||||||
you'll need quotes for it to parse correctly:
|
|
||||||
|
```bash
|
||||||
|
borgmatic create --override repositories="[{path: test.borg, label: test}]"
|
||||||
|
```
|
||||||
|
|
||||||
|
If your override value contains characters like colons or spaces, then you'll
|
||||||
|
need to use quotes for it to parse correctly.
|
||||||
|
|
||||||
|
Another example:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
borgmatic create --override repositories="['user@server:test.borg']"
|
borgmatic create --override repositories="['user@server:test.borg']"
|
||||||
|
@ -518,16 +526,12 @@ borgmatic create --override repositories="['user@server:test.borg']"
|
||||||
There is not currently a way to override a single element of a list without
|
There is not currently a way to override a single element of a list without
|
||||||
replacing the whole list.
|
replacing the whole list.
|
||||||
|
|
||||||
Note that if you override an option of the list type (like
|
Using the `[ ]` list syntax is required when overriding an option of the list
|
||||||
`location.repositories`), you do need to use the `[ ]` list syntax. See the
|
type (like `location.repositories`). See the [configuration
|
||||||
[configuration
|
|
||||||
reference](https://torsion.org/borgmatic/docs/reference/configuration/) for
|
reference](https://torsion.org/borgmatic/docs/reference/configuration/) for
|
||||||
which options are list types. (YAML list values look like `- this` with an
|
which options are list types. (YAML list values look like `- this` with an
|
||||||
indentation and a leading dash.)
|
indentation and a leading dash.)
|
||||||
|
|
||||||
Be sure to quote your overrides if they contain spaces or other characters
|
|
||||||
that your shell may interpret.
|
|
||||||
|
|
||||||
An alternate to command-line overrides is passing in your values via
|
An alternate to command-line overrides is passing in your values via
|
||||||
[environment
|
[environment
|
||||||
variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).
|
variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).
|
||||||
|
@ -540,8 +544,7 @@ tool is borgmatic's support for defining custom constants. This is similar to
|
||||||
the [variable interpolation
|
the [variable interpolation
|
||||||
feature](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation)
|
feature](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation)
|
||||||
for command hooks, but the constants feature lets you substitute your own
|
for command hooks, but the constants feature lets you substitute your own
|
||||||
custom values into anywhere in the entire configuration file. (Constants don't
|
custom values into any option values in the entire configuration file.
|
||||||
work across includes or separate configuration files though.)
|
|
||||||
|
|
||||||
Here's an example usage:
|
Here's an example usage:
|
||||||
|
|
||||||
|
@ -564,10 +567,15 @@ forget to specify the section (like `location:` or `storage:`) that any option
|
||||||
is in.
|
is in.
|
||||||
|
|
||||||
In this example, when borgmatic runs, all instances of `{user}` get replaced
|
In this example, when borgmatic runs, all instances of `{user}` get replaced
|
||||||
with `foo` and all instances of `{archive-prefix}` get replaced with `bar-`.
|
with `foo` and all instances of `{archive_prefix}` get replaced with `bar`.
|
||||||
(And in this particular example, `{now}` doesn't get replaced with anything,
|
And `{now}` doesn't get replaced with anything, but gets passed directly to
|
||||||
but gets passed directly to Borg.) After substitution, the logical result
|
Borg, which has its own
|
||||||
looks something like this:
|
[placeholders](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-placeholders)
|
||||||
|
using the same syntax as borgmatic constants. So borgmatic options like
|
||||||
|
`archive_name_format` that get passed directly to Borg can use either Borg
|
||||||
|
placeholders or borgmatic constants or both!
|
||||||
|
|
||||||
|
After substitution, the logical result looks something like this:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
source_directories:
|
source_directories:
|
||||||
|
@ -579,5 +587,24 @@ source_directories:
|
||||||
archive_name_format: 'bar-{now}'
|
archive_name_format: 'bar-{now}'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Note that if you'd like to interpolate a constant into the beginning of a
|
||||||
|
value, you'll need to quote it. For instance, this won't work:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
source_directories:
|
||||||
|
- {my_home_directory}/.config # This will error!
|
||||||
|
```
|
||||||
|
|
||||||
|
Instead, do this:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
source_directories:
|
||||||
|
- "{my_home_directory}/.config"
|
||||||
|
```
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.5</span> Constants
|
||||||
|
work across includes, meaning you can define a constant and then include a
|
||||||
|
separate configuration file that uses that constant.
|
||||||
|
|
||||||
An alternate to constants is passing in your values via [environment
|
An alternate to constants is passing in your values via [environment
|
||||||
variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).
|
variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).
|
||||||
|
|
|
@ -36,25 +36,24 @@ below for how to configure this.
|
||||||
|
|
||||||
### Third-party monitoring services
|
### Third-party monitoring services
|
||||||
|
|
||||||
borgmatic integrates with monitoring services like
|
borgmatic integrates with these monitoring services and libraries, pinging
|
||||||
[Healthchecks](https://healthchecks.io/), [Cronitor](https://cronitor.io),
|
them as backups happen:
|
||||||
[Cronhub](https://cronhub.io), [PagerDuty](https://www.pagerduty.com/), and
|
|
||||||
[ntfy](https://ntfy.sh/) and pings these services whenever borgmatic runs.
|
* [Healthchecks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
|
||||||
That way, you'll receive an alert when something goes wrong or (for certain
|
* [Cronitor](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook)
|
||||||
hooks) the service doesn't hear from borgmatic for a configured interval. See
|
* [Cronhub](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook)
|
||||||
[Healthchecks
|
* [PagerDuty](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook)
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook),
|
* [ntfy](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook)
|
||||||
[Cronitor
|
* [Grafana Loki](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook)
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook),
|
* [Apprise](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook)
|
||||||
[Cronhub
|
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook),
|
The idea is that you'll receive an alert when something goes wrong or when the
|
||||||
[PagerDuty
|
service doesn't hear from borgmatic for a configured interval (if supported).
|
||||||
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook),
|
See the documentation links above for configuration information.
|
||||||
and [ntfy hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook)
|
|
||||||
below for how to configure this.
|
While these services and libraries offer different features, you probably only
|
||||||
|
need to use one of them at most.
|
||||||
|
|
||||||
While these services offer different features, you probably only need to use
|
|
||||||
one of them at most.
|
|
||||||
|
|
||||||
### Third-party monitoring software
|
### Third-party monitoring software
|
||||||
|
|
||||||
|
@ -102,7 +101,7 @@ script to handle the alerting:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
on_error:
|
on_error:
|
||||||
- send-text-message.sh "{configuration_filename}" "{repository}"
|
- send-text-message.sh {configuration_filename} {repository}
|
||||||
```
|
```
|
||||||
|
|
||||||
In this example, when the error occurs, borgmatic interpolates runtime values
|
In this example, when the error occurs, borgmatic interpolates runtime values
|
||||||
|
@ -125,11 +124,32 @@ actions. borgmatic does not run `on_error` hooks if an error occurs within a
|
||||||
documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/),
|
documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/),
|
||||||
especially the security information.
|
especially the security information.
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.7</span> borgmatic
|
||||||
|
automatically escapes these interpolated values to prevent shell injection
|
||||||
|
attacks. One implication of this change is that you shouldn't wrap the
|
||||||
|
interpolated values in your own quotes, as that will interfere with the
|
||||||
|
quoting performed by borgmatic and result in your command receiving incorrect
|
||||||
|
arguments. For instance, this won't work:
|
||||||
|
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
on_error:
|
||||||
|
# Don't do this! It won't work, as the {error} value is already quoted.
|
||||||
|
- send-text-message.sh "Uh oh: {error}"
|
||||||
|
```
|
||||||
|
|
||||||
|
Do this instead:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
on_error:
|
||||||
|
- send-text-message.sh {error}
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
## Healthchecks hook
|
## Healthchecks hook
|
||||||
|
|
||||||
[Healthchecks](https://healthchecks.io/) is a service that provides "instant
|
[Healthchecks](https://healthchecks.io/) is a service that provides "instant
|
||||||
alerts when your cron jobs fail silently", and borgmatic has built-in
|
alerts when your cron jobs fail silently," and borgmatic has built-in
|
||||||
integration with it. Once you create a Healthchecks account and project on
|
integration with it. Once you create a Healthchecks account and project on
|
||||||
their site, all you need to do is configure borgmatic with the unique "Ping
|
their site, all you need to do is configure borgmatic with the unique "Ping
|
||||||
URL" for your project. Here's an example:
|
URL" for your project. Here's an example:
|
||||||
|
@ -143,22 +163,20 @@ healthchecks:
|
||||||
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
||||||
this option in the `hooks:` section of your configuration.
|
this option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
With this hook in place, borgmatic pings your Healthchecks project when a
|
With this configuration, borgmatic pings your Healthchecks project when a
|
||||||
backup begins, ends, or errors. Specifically, after the <a
|
backup begins, ends, or errors, but only when any of the `create`, `prune`,
|
||||||
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
|
`compact`, or `check` actions are run.
|
||||||
hooks</a> run, borgmatic lets Healthchecks know that it has started if any of
|
|
||||||
the `create`, `prune`, `compact`, or `check` actions are run.
|
|
||||||
|
|
||||||
Then, if the actions complete successfully, borgmatic notifies Healthchecks of
|
Then, if the actions complete successfully, borgmatic notifies Healthchecks of
|
||||||
the success after the `after_backup` hooks run and includes borgmatic logs in
|
the success and includes borgmatic logs in the payload data sent to
|
||||||
the payload data sent to Healthchecks. This means that borgmatic logs show up
|
Healthchecks. This means that borgmatic logs show up in the Healthchecks UI,
|
||||||
in the Healthchecks UI, although be aware that Healthchecks currently has a
|
although be aware that Healthchecks currently has a 100-kilobyte limit for the
|
||||||
10-kilobyte limit for the logs in each ping.
|
logs in each ping.
|
||||||
|
|
||||||
If an error occurs during any action or hook, borgmatic notifies Healthchecks
|
If an error occurs during any action or hook, borgmatic notifies Healthchecks,
|
||||||
after the `on_error` hooks run, also tacking on logs including the error
|
also tacking on logs including the error itself. But the logs are only
|
||||||
itself. But the logs are only included for errors that occur when a `create`,
|
included for errors that occur when a `create`, `prune`, `compact`, or `check`
|
||||||
`prune`, `compact`, or `check` action is run.
|
action is run.
|
||||||
|
|
||||||
You can customize the verbosity of the logs that are sent to Healthchecks with
|
You can customize the verbosity of the logs that are sent to Healthchecks with
|
||||||
borgmatic's `--monitoring-verbosity` flag. The `--list` and `--stats` flags
|
borgmatic's `--monitoring-verbosity` flag. The `--list` and `--stats` flags
|
||||||
|
@ -175,7 +193,7 @@ or it doesn't hear from borgmatic for a certain period of time.
|
||||||
## Cronitor hook
|
## Cronitor hook
|
||||||
|
|
||||||
[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
|
[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
|
||||||
for websites, services and APIs", and borgmatic has built-in
|
for websites, services and APIs," and borgmatic has built-in
|
||||||
integration with it. Once you create a Cronitor account and cron job monitor on
|
integration with it. Once you create a Cronitor account and cron job monitor on
|
||||||
their site, all you need to do is configure borgmatic with the unique "Ping
|
their site, all you need to do is configure borgmatic with the unique "Ping
|
||||||
API URL" for your monitor. Here's an example:
|
API URL" for your monitor. Here's an example:
|
||||||
|
@ -189,14 +207,10 @@ cronitor:
|
||||||
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
||||||
this option in the `hooks:` section of your configuration.
|
this option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
With this hook in place, borgmatic pings your Cronitor monitor when a backup
|
With this configuration, borgmatic pings your Cronitor monitor when a backup
|
||||||
begins, ends, or errors. Specifically, after the <a
|
begins, ends, or errors, but only when any of the `prune`, `compact`,
|
||||||
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
|
`create`, or `check` actions are run. Then, if the actions complete
|
||||||
hooks</a> run, borgmatic lets Cronitor know that it has started if any of the
|
successfully or errors, borgmatic notifies Cronitor accordingly.
|
||||||
`prune`, `compact`, `create`, or `check` actions are run. Then, if the actions
|
|
||||||
complete successfully, borgmatic notifies Cronitor of the success after the
|
|
||||||
`after_backup` hooks run. And if an error occurs during any action or hook,
|
|
||||||
borgmatic notifies Cronitor after the `on_error` hooks run.
|
|
||||||
|
|
||||||
You can configure Cronitor to notify you by a [variety of
|
You can configure Cronitor to notify you by a [variety of
|
||||||
mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
|
mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
|
||||||
|
@ -206,7 +220,7 @@ or it doesn't hear from borgmatic for a certain period of time.
|
||||||
## Cronhub hook
|
## Cronhub hook
|
||||||
|
|
||||||
[Cronhub](https://cronhub.io/) provides "instant alerts when any of your
|
[Cronhub](https://cronhub.io/) provides "instant alerts when any of your
|
||||||
background jobs fail silently or run longer than expected", and borgmatic has
|
background jobs fail silently or run longer than expected," and borgmatic has
|
||||||
built-in integration with it. Once you create a Cronhub account and monitor on
|
built-in integration with it. Once you create a Cronhub account and monitor on
|
||||||
their site, all you need to do is configure borgmatic with the unique "Ping
|
their site, all you need to do is configure borgmatic with the unique "Ping
|
||||||
URL" for your monitor. Here's an example:
|
URL" for your monitor. Here's an example:
|
||||||
|
@ -220,14 +234,10 @@ cronhub:
|
||||||
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
||||||
this option in the `hooks:` section of your configuration.
|
this option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
With this hook in place, borgmatic pings your Cronhub monitor when a backup
|
With this configuration, borgmatic pings your Cronhub monitor when a backup
|
||||||
begins, ends, or errors. Specifically, after the <a
|
begins, ends, or errors, but only when any of the `prune`, `compact`,
|
||||||
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
|
`create`, or `check` actions are run. Then, if the actions complete
|
||||||
hooks</a> run, borgmatic lets Cronhub know that it has started if any of the
|
successfully or errors, borgmatic notifies Cronhub accordingly.
|
||||||
`prune`, `compact`, `create`, or `check` actions are run. Then, if the actions
|
|
||||||
complete successfully, borgmatic notifies Cronhub of the success after the
|
|
||||||
`after_backup` hooks run. And if an error occurs during any action or hook,
|
|
||||||
borgmatic notifies Cronhub after the `on_error` hooks run.
|
|
||||||
|
|
||||||
Note that even though you configure borgmatic with the "start" variant of the
|
Note that even though you configure borgmatic with the "start" variant of the
|
||||||
ping URL, borgmatic substitutes the correct state into the URL when pinging
|
ping URL, borgmatic substitutes the correct state into the URL when pinging
|
||||||
|
@ -265,11 +275,10 @@ pagerduty:
|
||||||
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
||||||
this option in the `hooks:` section of your configuration.
|
this option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
With this hook in place, borgmatic creates a PagerDuty event for your service
|
With this configuration, borgmatic creates a PagerDuty event for your service
|
||||||
whenever backups fail. Specifically, if an error occurs during a `create`,
|
whenever backups fail, but only when any of the `create`, `prune`, `compact`,
|
||||||
`prune`, `compact`, or `check` action, borgmatic sends an event to PagerDuty
|
or `check` actions are run. Note that borgmatic does not contact PagerDuty
|
||||||
before the `on_error` hooks run. Note that borgmatic does not contact
|
when a backup starts or when it ends without error.
|
||||||
PagerDuty when a backup starts or ends without error.
|
|
||||||
|
|
||||||
You can configure PagerDuty to notify you by a [variety of
|
You can configure PagerDuty to notify you by a [variety of
|
||||||
mechanisms](https://support.pagerduty.com/docs/notifications) when backups
|
mechanisms](https://support.pagerduty.com/docs/notifications) when backups
|
||||||
|
@ -281,28 +290,30 @@ us](https://torsion.org/borgmatic/#support-and-contributing).
|
||||||
|
|
||||||
## ntfy hook
|
## ntfy hook
|
||||||
|
|
||||||
[ntfy](https://ntfy.sh) is a free, simple, service (either hosted or self-hosted)
|
<span class="minilink minilink-addedin">New in version 1.6.3</span>
|
||||||
which offers simple pub/sub push notifications to multiple platforms including
|
[ntfy](https://ntfy.sh) is a free, simple, service (either hosted or
|
||||||
[web](https://ntfy.sh/stats), [Android](https://play.google.com/store/apps/details?id=io.heckel.ntfy)
|
self-hosted) which offers simple pub/sub push notifications to multiple
|
||||||
and [iOS](https://apps.apple.com/us/app/ntfy/id1625396347).
|
platforms including [web](https://ntfy.sh/stats),
|
||||||
|
[Android](https://play.google.com/store/apps/details?id=io.heckel.ntfy) and
|
||||||
|
[iOS](https://apps.apple.com/us/app/ntfy/id1625396347).
|
||||||
|
|
||||||
Since push notifications for regular events might soon become quite annoying,
|
Since push notifications for regular events might soon become quite annoying,
|
||||||
this hook only fires on any errors by default in order to instantly alert you to issues.
|
this hook only fires on any errors by default in order to instantly alert you
|
||||||
The `states` list can override this.
|
to issues. The `states` list can override this. Each state can have its own
|
||||||
|
custom messages, priorities and tags or, if none are provided, will use the
|
||||||
|
default.
|
||||||
|
|
||||||
As ntfy is unauthenticated, it isn't a suitable channel for any private information
|
An example configuration is shown here with all the available options,
|
||||||
so the default messages are intentionally generic. These can be overridden, depending
|
including [priorities](https://ntfy.sh/docs/publish/#message-priority) and
|
||||||
on your risk assessment. Each `state` can have its own custom messages, priorities and tags
|
|
||||||
or, if none are provided, will use the default.
|
|
||||||
|
|
||||||
An example configuration is shown here, with all the available options, including
|
|
||||||
[priorities](https://ntfy.sh/docs/publish/#message-priority) and
|
|
||||||
[tags](https://ntfy.sh/docs/publish/#tags-emojis):
|
[tags](https://ntfy.sh/docs/publish/#tags-emojis):
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
ntfy:
|
ntfy:
|
||||||
topic: my-unique-topic
|
topic: my-unique-topic
|
||||||
server: https://ntfy.my-domain.com
|
server: https://ntfy.my-domain.com
|
||||||
|
username: myuser
|
||||||
|
password: secret
|
||||||
|
|
||||||
start:
|
start:
|
||||||
title: A borgmatic backup started
|
title: A borgmatic backup started
|
||||||
message: Watch this space...
|
message: Watch this space...
|
||||||
|
@ -327,6 +338,173 @@ ntfy:
|
||||||
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
||||||
the `ntfy:` option in the `hooks:` section of your configuration.
|
the `ntfy:` option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.9</span> Instead of
|
||||||
|
`username`/`password`, you can specify an [ntfy access
|
||||||
|
token](https://docs.ntfy.sh/config/#access-tokens):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
ntfy:
|
||||||
|
topic: my-unique-topic
|
||||||
|
server: https://ntfy.my-domain.com
|
||||||
|
access_token: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2
|
||||||
|
````
|
||||||
|
|
||||||
|
## Loki hook
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.3</span> [Grafana
|
||||||
|
Loki](https://grafana.com/oss/loki/) is a "horizontally scalable, highly
|
||||||
|
available, multi-tenant log aggregation system inspired by Prometheus."
|
||||||
|
borgmatic has built-in integration with Loki, sending both backup status and
|
||||||
|
borgmatic logs.
|
||||||
|
|
||||||
|
You can configure borgmatic to use either a [self-hosted Loki
|
||||||
|
instance](https://grafana.com/docs/loki/latest/installation/) or [a Grafana
|
||||||
|
Cloud account](https://grafana.com/auth/sign-up/create-user). Start by setting
|
||||||
|
your Loki API push URL. Here's an example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
loki:
|
||||||
|
url: http://localhost:3100/loki/api/v1/push
|
||||||
|
```
|
||||||
|
|
||||||
|
With this configuration, borgmatic sends its logs to your Loki instance as any
|
||||||
|
of the `prune`, `compact`, `create`, or `check` actions are run. Then, after
|
||||||
|
the actions complete, borgmatic notifies Loki of success or failure.
|
||||||
|
|
||||||
|
This hook supports sending arbitrary labels to Loki. For instance:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
loki:
|
||||||
|
url: http://localhost:3100/loki/api/v1/push
|
||||||
|
|
||||||
|
labels:
|
||||||
|
app: borgmatic
|
||||||
|
hostname: example.org
|
||||||
|
```
|
||||||
|
|
||||||
|
There are also a few placeholders you can optionally use as label values:
|
||||||
|
|
||||||
|
* `__config`: name of the borgmatic configuration file
|
||||||
|
* `__config_path`: full path of the borgmatic configuration file
|
||||||
|
* `__hostname`: the local machine hostname
|
||||||
|
|
||||||
|
These placeholders are only substituted for the whole label value, not
|
||||||
|
interpolated into a larger string. For instance:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
loki:
|
||||||
|
url: http://localhost:3100/loki/api/v1/push
|
||||||
|
|
||||||
|
labels:
|
||||||
|
app: borgmatic
|
||||||
|
config: __config
|
||||||
|
hostname: __hostname
|
||||||
|
```
|
||||||
|
|
||||||
|
Also check out this [Loki dashboard for
|
||||||
|
borgmatic](https://grafana.com/grafana/dashboards/20736-borgmatic-logs/) if
|
||||||
|
you'd like to see your backup logs and statistics in one place.
|
||||||
|
|
||||||
|
|
||||||
|
## Apprise hook
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.4</span>
|
||||||
|
[Apprise](https://github.com/caronc/apprise/wiki) is a local notification library
|
||||||
|
that "allows you to send a notification to almost all of the most popular
|
||||||
|
[notification services](https://github.com/caronc/apprise/wiki) available to
|
||||||
|
us today such as: Telegram, Discord, Slack, Amazon SNS, Gotify, etc."
|
||||||
|
|
||||||
|
Depending on how you installed borgmatic, it may not have come with Apprise.
|
||||||
|
For instance, if you originally [installed borgmatic with
|
||||||
|
pipx](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#installation),
|
||||||
|
run the following to install Apprise so borgmatic can use it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo pipx uninstall borgmatic
|
||||||
|
sudo pipx install borgmatic[Apprise]
|
||||||
|
```
|
||||||
|
|
||||||
|
Omit `sudo` if borgmatic is installed as a non-root user.
|
||||||
|
|
||||||
|
Once Apprise is installed, configure borgmatic to notify one or more [Apprise
|
||||||
|
services](https://github.com/caronc/apprise/wiki). For example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
apprise:
|
||||||
|
services:
|
||||||
|
- url: gotify://hostname/token
|
||||||
|
label: gotify
|
||||||
|
- url: mastodons://access_key@hostname/@user
|
||||||
|
label: mastodon
|
||||||
|
states:
|
||||||
|
- start
|
||||||
|
- finish
|
||||||
|
- fail
|
||||||
|
```
|
||||||
|
|
||||||
|
With this configuration, borgmatic pings each of the configured Apprise
|
||||||
|
services when a backup begins, ends, or errors, but only when any of the
|
||||||
|
`prune`, `compact`, `create`, or `check` actions are run. (By default, if
|
||||||
|
`states` is not specified, Apprise services are only pinged on error.)
|
||||||
|
|
||||||
|
You can optionally customize the contents of the default messages sent to
|
||||||
|
these services:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
apprise:
|
||||||
|
services:
|
||||||
|
- url: gotify://hostname/token
|
||||||
|
label: gotify
|
||||||
|
start:
|
||||||
|
title: Ping!
|
||||||
|
body: Starting backup process.
|
||||||
|
finish:
|
||||||
|
title: Ping!
|
||||||
|
body: Backups successfully made.
|
||||||
|
fail:
|
||||||
|
title: Ping!
|
||||||
|
body: Your backups have failed.
|
||||||
|
states:
|
||||||
|
- start
|
||||||
|
- finish
|
||||||
|
- fail
|
||||||
|
```
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.9</span> borgmatic
|
||||||
|
logs are automatically included in the body data sent to your Apprise services
|
||||||
|
when a backup finishes or fails.
|
||||||
|
|
||||||
|
You can customize the verbosity of the logs that are sent with borgmatic's
|
||||||
|
`--monitoring-verbosity` flag. The `--list` and `--stats` flags may also be of
|
||||||
|
use. See `borgmatic create --help` for more information.
|
||||||
|
|
||||||
|
If you don't want any logs sent, you can disable this feature by setting
|
||||||
|
`send_logs` to `false`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
apprise:
|
||||||
|
services:
|
||||||
|
- url: gotify://hostname/token
|
||||||
|
label: gotify
|
||||||
|
send_logs: false
|
||||||
|
```
|
||||||
|
|
||||||
|
Or to limit the size of logs sent to Apprise services:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
apprise:
|
||||||
|
services:
|
||||||
|
- url: gotify://hostname/token
|
||||||
|
label: gotify
|
||||||
|
logs_size_limit: 500
|
||||||
|
```
|
||||||
|
|
||||||
|
This may be necessary for some services that reject large requests.
|
||||||
|
|
||||||
|
See the [configuration
|
||||||
|
reference](https://torsion.org/borgmatic/docs/reference/configuration/) for
|
||||||
|
details.
|
||||||
|
|
||||||
|
|
||||||
## Scripting borgmatic
|
## Scripting borgmatic
|
||||||
|
|
||||||
|
|
|
@ -5,13 +5,31 @@ eleventyNavigation:
|
||||||
parent: How-to guides
|
parent: How-to guides
|
||||||
order: 2
|
order: 2
|
||||||
---
|
---
|
||||||
## Environment variable interpolation
|
## Providing passwords and secrets to borgmatic
|
||||||
|
|
||||||
If you want to use a Borg repository passphrase or database passwords with
|
If you want to use a Borg repository passphrase or database passwords with
|
||||||
borgmatic, you can set them directly in your borgmatic configuration file,
|
borgmatic, you can set them directly in your borgmatic configuration file,
|
||||||
treating those secrets like any other option value. But if you'd rather store
|
treating those secrets like any other option value. For instance, you can
|
||||||
them outside of borgmatic, whether for convenience or security reasons, read
|
specify your Borg passhprase with:
|
||||||
on.
|
|
||||||
|
```yaml
|
||||||
|
encryption_passphrase: yourpassphrase
|
||||||
|
```
|
||||||
|
|
||||||
|
But if you'd rather store them outside of borgmatic, whether for convenience
|
||||||
|
or security reasons, read on.
|
||||||
|
|
||||||
|
### Delegating to another application
|
||||||
|
|
||||||
|
borgmatic supports calling another application such as a password manager to
|
||||||
|
obtain the Borg passphrase to a repository.
|
||||||
|
|
||||||
|
For example, to ask the *Pass* password manager to provide the passphrase:
|
||||||
|
```yaml
|
||||||
|
encryption_passcommand: pass path/to/borg-repokey
|
||||||
|
```
|
||||||
|
|
||||||
|
### Environment variable interpolation
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">New in version 1.6.4</span> borgmatic
|
<span class="minilink minilink-addedin">New in version 1.6.4</span> borgmatic
|
||||||
supports interpolating arbitrary environment variables directly into option
|
supports interpolating arbitrary environment variables directly into option
|
||||||
|
@ -20,14 +38,14 @@ pull your repository passphrase, your database passwords, or any other option
|
||||||
values from environment variables. For instance:
|
values from environment variables. For instance:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
encryption_passphrase: ${MY_PASSPHRASE}
|
encryption_passphrase: ${YOUR_PASSPHRASE}
|
||||||
```
|
```
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
||||||
this option in the `storage:` section of your configuration.
|
this option in the `storage:` section of your configuration.
|
||||||
|
|
||||||
This uses the `MY_PASSPHRASE` environment variable as your encryption
|
This uses the `YOUR_PASSPHRASE` environment variable as your encryption
|
||||||
passphrase. Note that the `{` `}` brackets are required. `$MY_PASSPHRASE` by
|
passphrase. Note that the `{` `}` brackets are required. `$YOUR_PASSPHRASE` by
|
||||||
itself will not work.
|
itself will not work.
|
||||||
|
|
||||||
In the case of `encryption_passphrase` in particular, an alternate approach
|
In the case of `encryption_passphrase` in particular, an alternate approach
|
||||||
|
@ -42,30 +60,31 @@ the same approach applies. For example:
|
||||||
```yaml
|
```yaml
|
||||||
postgresql_databases:
|
postgresql_databases:
|
||||||
- name: users
|
- name: users
|
||||||
password: ${MY_DATABASE_PASSWORD}
|
password: ${YOUR_DATABASE_PASSWORD}
|
||||||
```
|
```
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
|
||||||
this option in the `hooks:` section of your configuration.
|
this option in the `hooks:` section of your configuration.
|
||||||
|
|
||||||
This uses the `MY_DATABASE_PASSWORD` environment variable as your database
|
This uses the `YOUR_DATABASE_PASSWORD` environment variable as your database
|
||||||
password.
|
password.
|
||||||
|
|
||||||
|
|
||||||
### Interpolation defaults
|
#### Interpolation defaults
|
||||||
|
|
||||||
If you'd like to set a default for your environment variables, you can do so with the following syntax:
|
If you'd like to set a default for your environment variables, you can do so
|
||||||
|
with the following syntax:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
encryption_passphrase: ${MY_PASSPHRASE:-defaultpass}
|
encryption_passphrase: ${YOUR_PASSPHRASE:-defaultpass}
|
||||||
```
|
```
|
||||||
|
|
||||||
Here, "`defaultpass`" is the default passphrase if the `MY_PASSPHRASE`
|
Here, "`defaultpass`" is the default passphrase if the `YOUR_PASSPHRASE`
|
||||||
environment variable is not set. Without a default, if the environment
|
environment variable is not set. Without a default, if the environment
|
||||||
variable doesn't exist, borgmatic will error.
|
variable doesn't exist, borgmatic will error.
|
||||||
|
|
||||||
|
|
||||||
### Disabling interpolation
|
#### Disabling interpolation
|
||||||
|
|
||||||
To disable this environment variable interpolation feature entirely, you can
|
To disable this environment variable interpolation feature entirely, you can
|
||||||
pass the `--no-environment-interpolation` flag on the command-line.
|
pass the `--no-environment-interpolation` flag on the command-line.
|
||||||
|
@ -78,7 +97,7 @@ can escape it with a backslash. For instance, if your password is literally
|
||||||
encryption_passphrase: \${A}@!
|
encryption_passphrase: \${A}@!
|
||||||
```
|
```
|
||||||
|
|
||||||
### Related features
|
## Related features
|
||||||
|
|
||||||
Another way to override particular options within a borgmatic configuration
|
Another way to override particular options within a borgmatic configuration
|
||||||
file is to use a [configuration
|
file is to use a [configuration
|
||||||
|
@ -90,3 +109,9 @@ Additionally, borgmatic action hooks support their own [variable
|
||||||
interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation),
|
interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation),
|
||||||
although in that case it's for particular borgmatic runtime values rather than
|
although in that case it's for particular borgmatic runtime values rather than
|
||||||
(only) environment variables.
|
(only) environment variables.
|
||||||
|
|
||||||
|
Lastly, if you do want to specify your passhprase directly within borgmatic
|
||||||
|
configuration, but you'd like to keep it in a separate file from your main
|
||||||
|
configuration, you can [use a configuration include or a merge
|
||||||
|
include](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-includes)
|
||||||
|
to pull in an external password.
|
||||||
|
|
|
@ -7,74 +7,70 @@ eleventyNavigation:
|
||||||
---
|
---
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
Many users need to backup system files that require privileged access, so
|
### Prerequisites
|
||||||
these instructions install and run borgmatic as root. If you don't need to
|
|
||||||
backup such files, then you are welcome to install and run borgmatic as a
|
|
||||||
non-root user.
|
|
||||||
|
|
||||||
First, manually [install
|
First, [install
|
||||||
Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
|
Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
|
||||||
version 1.1. borgmatic does not install Borg automatically so as to avoid
|
version 1.1. borgmatic does not install Borg automatically so as to avoid
|
||||||
conflicts with existing Borg installations.
|
conflicts with existing Borg installations.
|
||||||
|
|
||||||
Then, download and install borgmatic as a [user site
|
Then, [install pipx](https://pypa.github.io/pipx/installation/) as the root
|
||||||
installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site)
|
user (with `sudo`) to make installing borgmatic easy without impacting other
|
||||||
by running the following command:
|
Python applications on your system. If you have trouble installing pipx with
|
||||||
|
pip, then you can install a system package instead. E.g. on Ubuntu or Debian,
|
||||||
|
run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo pip3 install --user --upgrade borgmatic
|
sudo apt update
|
||||||
|
sudo apt install pipx
|
||||||
```
|
```
|
||||||
|
|
||||||
This installs borgmatic and its commands at the `/root/.local/bin` path.
|
### Root install
|
||||||
|
|
||||||
Your pip binary may have a different name than "pip3". Make sure you're using
|
If you want to run borgmatic on a schedule with privileged access to your
|
||||||
Python 3.7+, as borgmatic does not support older versions of Python.
|
files, then you should install borgmatic as the root user by running the
|
||||||
|
following commands:
|
||||||
The next step is to ensure that borgmatic's commands available are on your
|
|
||||||
system `PATH`, so that you can run borgmatic:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
echo export 'PATH="$PATH:/root/.local/bin"' >> ~/.bashrc
|
sudo pipx ensurepath
|
||||||
source ~/.bashrc
|
sudo pipx install borgmatic
|
||||||
```
|
```
|
||||||
|
|
||||||
This adds `/root/.local/bin` to your non-root user's system `PATH`.
|
Check whether this worked with:
|
||||||
|
|
||||||
If you're using a command shell other than Bash, you may need to use different
|
|
||||||
commands here.
|
|
||||||
|
|
||||||
You can check whether all of this worked with:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo borgmatic --version
|
sudo su -
|
||||||
|
borgmatic --version
|
||||||
```
|
```
|
||||||
|
|
||||||
If borgmatic is properly installed, that should output your borgmatic version.
|
If borgmatic is properly installed, that should output your borgmatic version.
|
||||||
|
And if you'd also like `sudo borgmatic` to work, keep reading!
|
||||||
As an alternative to adding the path to `~/.bashrc` file, if you're using sudo
|
|
||||||
to run borgmatic, you can configure [sudo's
|
|
||||||
`secure_path` option](https://man.archlinux.org/man/sudoers.5) to include
|
|
||||||
borgmatic's path.
|
|
||||||
|
|
||||||
|
|
||||||
### Global install option
|
### Non-root install
|
||||||
|
|
||||||
If you try the user site installation above and have problems making borgmatic
|
If you only want to run borgmatic as a non-root user (without privileged file
|
||||||
commands runnable on your system `PATH`, an alternate approach is to install
|
access) *or* you want to make `sudo borgmatic` work so borgmatic runs as root,
|
||||||
borgmatic globally.
|
then install borgmatic as a non-root user by running the following commands as
|
||||||
|
that user:
|
||||||
The following uninstalls borgmatic and then reinstalls it such that borgmatic
|
|
||||||
commands are on the default system `PATH`:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo pip3 uninstall borgmatic
|
pipx ensurepath
|
||||||
sudo pip3 install --upgrade borgmatic
|
pipx install borgmatic
|
||||||
```
|
```
|
||||||
|
|
||||||
The main downside of a global install is that borgmatic is less cleanly
|
This should work even if you've also installed borgmatic as the root user.
|
||||||
separated from the rest of your Python software, and there's the theoretical
|
|
||||||
possibility of library conflicts. But if you're okay with that, for instance
|
Check whether this worked with:
|
||||||
on a relatively dedicated system, then a global install can work out fine.
|
|
||||||
|
```bash
|
||||||
|
borgmatic --version
|
||||||
|
```
|
||||||
|
|
||||||
|
If borgmatic is properly installed, that should output your borgmatic version.
|
||||||
|
You can also try `sudo borgmatic --version` if you intend to run borgmatic
|
||||||
|
with `sudo`. If that doesn't work, you may need to update your [sudoers
|
||||||
|
`secure_path` option](https://wiki.archlinux.org/title/Sudo).
|
||||||
|
|
||||||
|
|
||||||
### Other ways to install
|
### Other ways to install
|
||||||
|
@ -286,6 +282,21 @@ due to things like file damage. For instance:
|
||||||
sudo borgmatic --verbosity 1 --list --stats
|
sudo borgmatic --verbosity 1 --list --stats
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Skipping actions
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.8.5</span> You can
|
||||||
|
configure borgmatic to skip running certain actions (default or otherwise).
|
||||||
|
For instance, to always skip the `compact` action when using [Borg's
|
||||||
|
append-only
|
||||||
|
mode](https://borgbackup.readthedocs.io/en/stable/usage/notes.html#append-only-mode-forbid-compaction),
|
||||||
|
set the `skip_actions` option:
|
||||||
|
|
||||||
|
```
|
||||||
|
skip_actions:
|
||||||
|
- compact
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
## Autopilot
|
## Autopilot
|
||||||
|
|
||||||
Running backups manually is good for validating your configuration, but I'm
|
Running backups manually is good for validating your configuration, but I'm
|
||||||
|
@ -395,8 +406,9 @@ source /usr/share/fish/vendor_completions.d/borgmatic.fish
|
||||||
borgmatic produces colored terminal output by default. It is disabled when a
|
borgmatic produces colored terminal output by default. It is disabled when a
|
||||||
non-interactive terminal is detected (like a cron job), or when you use the
|
non-interactive terminal is detected (like a cron job), or when you use the
|
||||||
`--json` flag. Otherwise, you can disable it by passing the `--no-color` flag,
|
`--json` flag. Otherwise, you can disable it by passing the `--no-color` flag,
|
||||||
setting the environment variable `PY_COLORS=False`, or setting the `color`
|
setting the environment variables `PY_COLORS=False` or `NO_COLOR=True`, or
|
||||||
option to `false` in the `output` section of configuration.
|
setting the `color` option to `false` in the `output` section of
|
||||||
|
configuration.
|
||||||
|
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
|
@ -3,30 +3,42 @@ title: How to upgrade borgmatic and Borg
|
||||||
eleventyNavigation:
|
eleventyNavigation:
|
||||||
key: 📦 Upgrade borgmatic/Borg
|
key: 📦 Upgrade borgmatic/Borg
|
||||||
parent: How-to guides
|
parent: How-to guides
|
||||||
order: 12
|
order: 13
|
||||||
---
|
---
|
||||||
## Upgrading borgmatic
|
## Upgrading borgmatic
|
||||||
|
|
||||||
In general, all you should need to do to upgrade borgmatic is run the
|
In general, all you should need to do to upgrade borgmatic if you've
|
||||||
following:
|
[installed it with
|
||||||
|
pipx](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#installation)
|
||||||
|
is to run the following:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo pip3 install --user --upgrade borgmatic
|
sudo pipx upgrade borgmatic
|
||||||
```
|
```
|
||||||
|
|
||||||
See below about special cases with old versions of borgmatic. Additionally, if
|
Omit `sudo` if you installed borgmatic as a non-root user. And if you
|
||||||
you installed borgmatic [without using `pip3 install
|
installed borgmatic *both* as root and as a non-root user, you'll need to
|
||||||
--user`](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#other-ways-to-install),
|
upgrade each installation independently.
|
||||||
then your upgrade process may be different.
|
|
||||||
|
If you originally installed borgmatic with `sudo pip3 install --user`, you can
|
||||||
|
uninstall it first with `sudo pip3 uninstall borgmatic` and then [install it
|
||||||
|
again with
|
||||||
|
pipx](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#installation),
|
||||||
|
which should better isolate borgmatic from your other Python applications.
|
||||||
|
|
||||||
|
But if you [installed borgmatic without pipx or
|
||||||
|
pip3](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#other-ways-to-install),
|
||||||
|
then your upgrade method may be different.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### Upgrading your configuration
|
### Upgrading your configuration
|
||||||
|
|
||||||
The borgmatic configuration file format is almost always backwards-compatible
|
The borgmatic configuration file format is usually backwards-compatible from
|
||||||
from release to release without any changes, but you may still want to update
|
release to release without any changes, but you may still want to update your
|
||||||
your configuration file when you upgrade to take advantage of new
|
configuration file when you upgrade to take advantage of new configuration
|
||||||
configuration options. This is completely optional. If you prefer, you can add
|
options or avoid old configuration from eventually becoming unsupported. If
|
||||||
new configuration options manually.
|
you prefer, you can add new configuration options manually.
|
||||||
|
|
||||||
If you do want to upgrade your configuration file to include new options, use
|
If you do want to upgrade your configuration file to include new options, use
|
||||||
the `borgmatic config generate` action with its optional `--source` flag that
|
the `borgmatic config generate` action with its optional `--source` flag that
|
||||||
|
@ -64,45 +76,10 @@ and, if desired, replace your original configuration file with it.
|
||||||
|
|
||||||
borgmatic changed its configuration file format in version 1.1.0 from
|
borgmatic changed its configuration file format in version 1.1.0 from
|
||||||
INI-style to YAML. This better supports validation and has a more natural way
|
INI-style to YAML. This better supports validation and has a more natural way
|
||||||
to express lists of values. To upgrade your existing configuration, first
|
to express lists of values. Modern versions of borgmatic no longer include
|
||||||
upgrade to the last version of borgmatic to support converting configuration:
|
support for upgrading configuration files this old, but feel free to [file a
|
||||||
borgmatic 1.7.14.
|
ticket](https://torsion.org/borgmatic/#issues) for help with upgrading any old
|
||||||
|
INI-style configuration files you may have.
|
||||||
As of version 1.1.0, borgmatic no longer supports Python 2. If you were
|
|
||||||
already running borgmatic with Python 3, then you can upgrade borgmatic
|
|
||||||
in-place:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo pip3 install --user --upgrade borgmatic==1.7.14
|
|
||||||
```
|
|
||||||
|
|
||||||
But if you were running borgmatic with Python 2, uninstall and reinstall instead:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo pip uninstall borgmatic
|
|
||||||
sudo pip3 install --user borgmatic==1.7.14
|
|
||||||
```
|
|
||||||
|
|
||||||
The pip binary names for different versions of Python can differ, so the above
|
|
||||||
commands may need some tweaking to work on your machine.
|
|
||||||
|
|
||||||
|
|
||||||
Once borgmatic is upgraded, run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo upgrade-borgmatic-config
|
|
||||||
```
|
|
||||||
|
|
||||||
That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
|
|
||||||
(by default) using the values from both your existing configuration and
|
|
||||||
excludes files. The new version of borgmatic will consume the YAML
|
|
||||||
configuration file instead of the old one.
|
|
||||||
|
|
||||||
Now you can upgrade to a newer version of borgmatic:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo pip3 install --user borgmatic
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
## Upgrading Borg
|
## Upgrading Borg
|
||||||
|
|
|
@ -21,5 +21,3 @@ version](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#configuration
|
||||||
```yaml
|
```yaml
|
||||||
{% include borgmatic/config.yaml %}
|
{% include borgmatic/config.yaml %}
|
||||||
```
|
```
|
||||||
|
|
||||||
Note that you can also [download this configuration
|
|
||||||
|
|
Binary file not shown.
After Width: | Height: | Size: 157 KiB |
Binary file not shown.
After Width: | Height: | Size: 15 KiB |
|
@ -32,16 +32,16 @@ RestrictSUIDSGID=yes
|
||||||
SystemCallArchitectures=native
|
SystemCallArchitectures=native
|
||||||
SystemCallFilter=@system-service
|
SystemCallFilter=@system-service
|
||||||
SystemCallErrorNumber=EPERM
|
SystemCallErrorNumber=EPERM
|
||||||
# To restrict write access further, change "ProtectSystem" to "strict" and uncomment
|
# To restrict write access further, change "ProtectSystem" to "strict" and
|
||||||
# "ReadWritePaths", "ReadOnlyPaths", "ProtectHome", and "BindPaths". Then add any local repository
|
# uncomment "ReadWritePaths", "TemporaryFileSystem", "BindPaths" and
|
||||||
# paths to the list of "ReadWritePaths" and local backup source paths to "ReadOnlyPaths". This
|
# "BindReadOnlyPaths". Then add any local repository paths to the list of
|
||||||
# leaves most of the filesystem read-only to borgmatic.
|
# "ReadWritePaths". This leaves most of the filesystem read-only to borgmatic.
|
||||||
ProtectSystem=full
|
ProtectSystem=full
|
||||||
# ReadWritePaths=-/mnt/my_backup_drive
|
# ReadWritePaths=-/mnt/my_backup_drive
|
||||||
# ReadOnlyPaths=-/var/lib/my_backup_source
|
|
||||||
# This will mount a tmpfs on top of /root and pass through needed paths
|
# This will mount a tmpfs on top of /root and pass through needed paths
|
||||||
# ProtectHome=tmpfs
|
# TemporaryFileSystem=/root:ro
|
||||||
# BindPaths=-/root/.cache/borg -/root/.config/borg -/root/.borgmatic
|
# BindPaths=-/root/.cache/borg -/root/.config/borg -/root/.borgmatic
|
||||||
|
# BindReadOnlyPaths=-/root/.ssh
|
||||||
|
|
||||||
# May interfere with running external programs within borgmatic hooks.
|
# May interfere with running external programs within borgmatic hooks.
|
||||||
CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_RAW
|
CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_RAW
|
||||||
|
|
|
@ -16,5 +16,7 @@ if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then
|
||||||
export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH"
|
export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \
|
docker-compose --file tests/end-to-end/docker-compose.yaml --progress quiet up --force-recreate \
|
||||||
--renew-anon-volumes --abort-on-container-exit
|
--renew-anon-volumes --detach
|
||||||
|
docker-compose --file tests/end-to-end/docker-compose.yaml --progress quiet attach tests
|
||||||
|
docker-compose --file tests/end-to-end/docker-compose.yaml --progress quiet down
|
|
@ -3,7 +3,7 @@
|
||||||
# This script installs test dependencies and runs all tests, including end-to-end tests. It
|
# This script installs test dependencies and runs all tests, including end-to-end tests. It
|
||||||
# is designed to run inside a test container, and presumes that other test infrastructure like
|
# is designed to run inside a test container, and presumes that other test infrastructure like
|
||||||
# databases are already running. Therefore, on a developer machine, you should not run this script
|
# databases are already running. Therefore, on a developer machine, you should not run this script
|
||||||
# directly. Instead, run scripts/run-end-to-end-dev-tests
|
# directly. Instead, run scripts/run-end-to-end-tests
|
||||||
#
|
#
|
||||||
# For more information, see:
|
# For more information, see:
|
||||||
# https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
|
# https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
|
||||||
|
@ -18,15 +18,12 @@ if [ -z "$TEST_CONTAINER" ]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \
|
apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \
|
||||||
py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite fish
|
py3-ruamel.yaml py3-ruamel.yaml.clib py3-yaml bash sqlite fish
|
||||||
# If certain dependencies of black are available in this version of Alpine, install them.
|
# If certain dependencies of black are available in this version of Alpine, install them.
|
||||||
apk add --no-cache py3-typed-ast py3-regex || true
|
apk add --no-cache py3-typed-ast py3-regex || true
|
||||||
python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 pymongo==4.4.1
|
python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1
|
||||||
pip3 install --ignore-installed tox==3.25.1
|
pip3 install --ignore-installed tox==4.11.3
|
||||||
export COVERAGE_FILE=/tmp/.coverage
|
export COVERAGE_FILE=/tmp/.coverage
|
||||||
|
|
||||||
if [ "$1" != "--end-to-end-only" ]; then
|
tox --workdir /tmp/.tox --sitepackages
|
||||||
tox --workdir /tmp/.tox --sitepackages
|
|
||||||
fi
|
|
||||||
|
|
||||||
tox --workdir /tmp/.tox --sitepackages -e end-to-end
|
tox --workdir /tmp/.tox --sitepackages -e end-to-end
|
||||||
|
|
7
setup.py
7
setup.py
|
@ -1,6 +1,6 @@
|
||||||
from setuptools import find_packages, setup
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
VERSION = '1.8.2'
|
VERSION = '1.8.12.dev0'
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
|
@ -33,9 +33,10 @@ setup(
|
||||||
'jsonschema',
|
'jsonschema',
|
||||||
'packaging',
|
'packaging',
|
||||||
'requests',
|
'requests',
|
||||||
'ruamel.yaml>0.15.0,<0.18.0',
|
'ruamel.yaml>0.15.0',
|
||||||
'setuptools',
|
'setuptools',
|
||||||
),
|
),
|
||||||
|
extras_require={"Apprise": ["apprise"]},
|
||||||
include_package_data=True,
|
include_package_data=True,
|
||||||
python_requires='>=3.7',
|
python_requires='>=3.8',
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,8 +1,10 @@
|
||||||
appdirs==1.4.4; python_version >= '3.8'
|
appdirs==1.4.4
|
||||||
attrs==22.2.0; python_version >= '3.8'
|
apprise==1.3.0
|
||||||
black==23.3.0; python_version >= '3.8'
|
attrs==22.2.0
|
||||||
|
black==24.3.0
|
||||||
|
certifi==2023.7.22
|
||||||
chardet==5.1.0
|
chardet==5.1.0
|
||||||
click==8.1.3; python_version >= '3.8'
|
click==8.1.3
|
||||||
codespell==2.2.4
|
codespell==2.2.4
|
||||||
colorama==0.4.6
|
colorama==0.4.6
|
||||||
coverage==7.2.3
|
coverage==7.2.3
|
||||||
|
@ -11,23 +13,22 @@ flake8-quotes==3.3.2
|
||||||
flake8-use-fstring==1.4
|
flake8-use-fstring==1.4
|
||||||
flake8-variables-names==0.0.5
|
flake8-variables-names==0.0.5
|
||||||
flexmock==0.11.3
|
flexmock==0.11.3
|
||||||
idna==3.4
|
idna==3.7
|
||||||
importlib_metadata==6.3.0; python_version < '3.8'
|
|
||||||
isort==5.12.0
|
isort==5.12.0
|
||||||
|
jsonschema==4.17.3
|
||||||
|
Markdown==3.4.1
|
||||||
mccabe==0.7.0
|
mccabe==0.7.0
|
||||||
packaging==23.1
|
packaging==23.1
|
||||||
|
pathspec==0.11.1
|
||||||
pluggy==1.0.0
|
pluggy==1.0.0
|
||||||
pathspec==0.11.1; python_version >= '3.8'
|
|
||||||
py==1.11.0
|
py==1.11.0
|
||||||
pycodestyle==2.10.0
|
pycodestyle==2.10.0
|
||||||
pyflakes==3.0.1
|
pyflakes==3.0.1
|
||||||
jsonschema==4.17.3
|
|
||||||
pytest==7.3.0
|
pytest==7.3.0
|
||||||
pytest-cov==4.0.0
|
pytest-cov==4.0.0
|
||||||
regex; python_version >= '3.8'
|
PyYAML>5.0.0
|
||||||
|
regex
|
||||||
requests==2.31.0
|
requests==2.31.0
|
||||||
ruamel.yaml>0.15.0,<0.18.0
|
ruamel.yaml>0.15.0
|
||||||
toml==0.10.2; python_version >= '3.8'
|
toml==0.10.2
|
||||||
typed-ast; python_version >= '3.8'
|
typed-ast
|
||||||
typing-extensions==4.5.0; python_version < '3.8'
|
|
||||||
zipp==3.15.0; python_version < '3.8'
|
|
||||||
|
|
|
@ -49,14 +49,13 @@ services:
|
||||||
environment:
|
environment:
|
||||||
TEST_CONTAINER: true
|
TEST_CONTAINER: true
|
||||||
volumes:
|
volumes:
|
||||||
- "../..:/app:ro"
|
- "../..:/app"
|
||||||
tmpfs:
|
tmpfs:
|
||||||
- "/app/borgmatic.egg-info"
|
- "/app/borgmatic.egg-info"
|
||||||
- "/app/build"
|
- "/app/build"
|
||||||
tty: true
|
tty: true
|
||||||
working_dir: /app
|
working_dir: /app
|
||||||
entrypoint: /app/scripts/run-full-tests
|
entrypoint: /app/scripts/run-full-tests
|
||||||
command: --end-to-end-only
|
|
||||||
depends_on:
|
depends_on:
|
||||||
- postgresql
|
- postgresql
|
||||||
- postgresql2
|
- postgresql2
|
||||||
|
|
|
@ -1,38 +0,0 @@
|
||||||
import ruamel.yaml
|
|
||||||
|
|
||||||
|
|
||||||
def test_dev_docker_compose_has_same_services_as_build_server_configuration():
|
|
||||||
'''
|
|
||||||
The end-to-end test configuration for local development and the build server's test
|
|
||||||
configuration use two different mechanisms for configuring and spinning up "services"—the
|
|
||||||
database containers upon which the end-to-end tests are reliant. The dev configuration uses
|
|
||||||
Docker Compose, while the Drone build server configuration uses its own similar-but-different
|
|
||||||
configuration file format.
|
|
||||||
|
|
||||||
Therefore, to ensure dev-build parity, these tests assert that the services are the same across
|
|
||||||
the dev and build configurations. This includes service name, container image, environment
|
|
||||||
variables, and commands.
|
|
||||||
|
|
||||||
This test only compares services and does not assert anything else about the respective testing
|
|
||||||
environments.
|
|
||||||
'''
|
|
||||||
yaml = ruamel.yaml.YAML(typ='safe')
|
|
||||||
dev_services = {
|
|
||||||
name: service
|
|
||||||
for name, service in yaml.load(open('tests/end-to-end/docker-compose.yaml').read())[
|
|
||||||
'services'
|
|
||||||
].items()
|
|
||||||
if name != 'tests'
|
|
||||||
}
|
|
||||||
build_server_services = tuple(yaml.load_all(open('.drone.yml').read()))[0]['services']
|
|
||||||
|
|
||||||
assert len(dev_services) == len(build_server_services)
|
|
||||||
|
|
||||||
for build_service in build_server_services:
|
|
||||||
dev_service = dev_services[build_service['name']]
|
|
||||||
assert dev_service['image'] == build_service['image']
|
|
||||||
assert dev_service['environment'] == build_service['environment']
|
|
||||||
|
|
||||||
if 'command' in dev_service or 'commands' in build_service:
|
|
||||||
assert len(build_service['commands']) <= 1
|
|
||||||
assert dev_service['command'] == build_service['commands'][0]
|
|
|
@ -0,0 +1,11 @@
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def test_borgmatic_command_with_invalid_flag_shows_error_but_not_traceback():
|
||||||
|
output = subprocess.run(
|
||||||
|
'borgmatic -v 2 --invalid'.split(' '), stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||||
|
).stdout.decode(sys.stdout.encoding)
|
||||||
|
|
||||||
|
assert 'Unrecognized argument' in output
|
||||||
|
assert 'Traceback' not in output
|
|
@ -32,6 +32,9 @@ def assert_command_does_not_duplicate_flags(command, *args, **kwargs):
|
||||||
flag_name: 1 for flag_name in flag_counts
|
flag_name: 1 for flag_name in flag_counts
|
||||||
}, f"Duplicate flags found in: {' '.join(command)}"
|
}, f"Duplicate flags found in: {' '.join(command)}"
|
||||||
|
|
||||||
|
if '--json' in command:
|
||||||
|
return '{}'
|
||||||
|
|
||||||
|
|
||||||
def fuzz_argument(arguments, argument_name):
|
def fuzz_argument(arguments, argument_name):
|
||||||
'''
|
'''
|
||||||
|
|
|
@ -13,8 +13,9 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
|
||||||
global_arguments = arguments['global']
|
global_arguments = arguments['global']
|
||||||
assert global_arguments.config_paths == config_paths
|
assert global_arguments.config_paths == config_paths
|
||||||
assert global_arguments.verbosity == 0
|
assert global_arguments.verbosity == 0
|
||||||
assert global_arguments.syslog_verbosity == 0
|
assert global_arguments.syslog_verbosity == -2
|
||||||
assert global_arguments.log_file_verbosity == 0
|
assert global_arguments.log_file_verbosity == 1
|
||||||
|
assert global_arguments.monitoring_verbosity == 1
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_multiple_config_flags_parses_as_list():
|
def test_parse_arguments_with_multiple_config_flags_parses_as_list():
|
||||||
|
@ -25,8 +26,9 @@ def test_parse_arguments_with_multiple_config_flags_parses_as_list():
|
||||||
global_arguments = arguments['global']
|
global_arguments = arguments['global']
|
||||||
assert global_arguments.config_paths == ['myconfig', 'otherconfig']
|
assert global_arguments.config_paths == ['myconfig', 'otherconfig']
|
||||||
assert global_arguments.verbosity == 0
|
assert global_arguments.verbosity == 0
|
||||||
assert global_arguments.syslog_verbosity == 0
|
assert global_arguments.syslog_verbosity == -2
|
||||||
assert global_arguments.log_file_verbosity == 0
|
assert global_arguments.log_file_verbosity == 1
|
||||||
|
assert global_arguments.monitoring_verbosity == 1
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_action_after_config_path_omits_action():
|
def test_parse_arguments_with_action_after_config_path_omits_action():
|
||||||
|
@ -71,8 +73,9 @@ def test_parse_arguments_with_verbosity_overrides_default():
|
||||||
global_arguments = arguments['global']
|
global_arguments = arguments['global']
|
||||||
assert global_arguments.config_paths == config_paths
|
assert global_arguments.config_paths == config_paths
|
||||||
assert global_arguments.verbosity == 1
|
assert global_arguments.verbosity == 1
|
||||||
assert global_arguments.syslog_verbosity == 0
|
assert global_arguments.syslog_verbosity == -2
|
||||||
assert global_arguments.log_file_verbosity == 0
|
assert global_arguments.log_file_verbosity == 1
|
||||||
|
assert global_arguments.monitoring_verbosity == 1
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_syslog_verbosity_overrides_default():
|
def test_parse_arguments_with_syslog_verbosity_overrides_default():
|
||||||
|
@ -85,6 +88,8 @@ def test_parse_arguments_with_syslog_verbosity_overrides_default():
|
||||||
assert global_arguments.config_paths == config_paths
|
assert global_arguments.config_paths == config_paths
|
||||||
assert global_arguments.verbosity == 0
|
assert global_arguments.verbosity == 0
|
||||||
assert global_arguments.syslog_verbosity == 2
|
assert global_arguments.syslog_verbosity == 2
|
||||||
|
assert global_arguments.log_file_verbosity == 1
|
||||||
|
assert global_arguments.monitoring_verbosity == 1
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_log_file_verbosity_overrides_default():
|
def test_parse_arguments_with_log_file_verbosity_overrides_default():
|
||||||
|
@ -96,8 +101,9 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default():
|
||||||
global_arguments = arguments['global']
|
global_arguments = arguments['global']
|
||||||
assert global_arguments.config_paths == config_paths
|
assert global_arguments.config_paths == config_paths
|
||||||
assert global_arguments.verbosity == 0
|
assert global_arguments.verbosity == 0
|
||||||
assert global_arguments.syslog_verbosity == 0
|
assert global_arguments.syslog_verbosity == -2
|
||||||
assert global_arguments.log_file_verbosity == -1
|
assert global_arguments.log_file_verbosity == -1
|
||||||
|
assert global_arguments.monitoring_verbosity == 1
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_single_override_parses():
|
def test_parse_arguments_with_single_override_parses():
|
||||||
|
@ -616,3 +622,16 @@ def test_parse_arguments_config_with_subaction_and_explicit_config_file_does_not
|
||||||
module.parse_arguments(
|
module.parse_arguments(
|
||||||
'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml'
|
'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_arguments_with_borg_action_and_dry_run_raises():
|
||||||
|
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
module.parse_arguments('--dry-run', 'borg', 'list')
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_arguments_with_borg_action_and_no_dry_run_does_not_raise():
|
||||||
|
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
|
||||||
|
|
||||||
|
module.parse_arguments('borg', 'list')
|
||||||
|
|
|
@ -10,7 +10,7 @@ from borgmatic.config import generate as module
|
||||||
|
|
||||||
def test_insert_newline_before_comment_does_not_raise():
|
def test_insert_newline_before_comment_does_not_raise():
|
||||||
field_name = 'foo'
|
field_name = 'foo'
|
||||||
config = module.yaml.comments.CommentedMap([(field_name, 33)])
|
config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)])
|
||||||
config.yaml_set_comment_before_after_key(key=field_name, before='Comment')
|
config.yaml_set_comment_before_after_key(key=field_name, before='Comment')
|
||||||
|
|
||||||
module.insert_newline_before_comment(config, field_name)
|
module.insert_newline_before_comment(config, field_name)
|
||||||
|
@ -125,14 +125,16 @@ def test_write_configuration_with_already_existing_directory_does_not_raise():
|
||||||
|
|
||||||
|
|
||||||
def test_add_comments_to_configuration_sequence_of_strings_does_not_raise():
|
def test_add_comments_to_configuration_sequence_of_strings_does_not_raise():
|
||||||
config = module.yaml.comments.CommentedSeq(['foo', 'bar'])
|
config = module.ruamel.yaml.comments.CommentedSeq(['foo', 'bar'])
|
||||||
schema = {'type': 'array', 'items': {'type': 'string'}}
|
schema = {'type': 'array', 'items': {'type': 'string'}}
|
||||||
|
|
||||||
module.add_comments_to_configuration_sequence(config, schema)
|
module.add_comments_to_configuration_sequence(config, schema)
|
||||||
|
|
||||||
|
|
||||||
def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
|
def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
|
||||||
config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
|
config = module.ruamel.yaml.comments.CommentedSeq(
|
||||||
|
[module.ruamel.yaml.comments.CommentedMap([('foo', 'yo')])]
|
||||||
|
)
|
||||||
schema = {
|
schema = {
|
||||||
'type': 'array',
|
'type': 'array',
|
||||||
'items': {'type': 'object', 'properties': {'foo': {'description': 'yo'}}},
|
'items': {'type': 'object', 'properties': {'foo': {'description': 'yo'}}},
|
||||||
|
@ -142,7 +144,9 @@ def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
|
||||||
|
|
||||||
|
|
||||||
def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise():
|
def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise():
|
||||||
config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
|
config = module.ruamel.yaml.comments.CommentedSeq(
|
||||||
|
[module.ruamel.yaml.comments.CommentedMap([('foo', 'yo')])]
|
||||||
|
)
|
||||||
schema = {'type': 'array', 'items': {'type': 'object', 'properties': {'foo': {}}}}
|
schema = {'type': 'array', 'items': {'type': 'object', 'properties': {'foo': {}}}}
|
||||||
|
|
||||||
module.add_comments_to_configuration_sequence(config, schema)
|
module.add_comments_to_configuration_sequence(config, schema)
|
||||||
|
@ -150,7 +154,7 @@ def test_add_comments_to_configuration_sequence_of_maps_without_description_does
|
||||||
|
|
||||||
def test_add_comments_to_configuration_object_does_not_raise():
|
def test_add_comments_to_configuration_object_does_not_raise():
|
||||||
# Ensure that it can deal with fields both in the schema and missing from the schema.
|
# Ensure that it can deal with fields both in the schema and missing from the schema.
|
||||||
config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
|
config = module.ruamel.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
|
||||||
schema = {
|
schema = {
|
||||||
'type': 'object',
|
'type': 'object',
|
||||||
'properties': {'foo': {'description': 'Foo'}, 'bar': {'description': 'Bar'}},
|
'properties': {'foo': {'description': 'Foo'}, 'bar': {'description': 'Bar'}},
|
||||||
|
@ -160,7 +164,7 @@ def test_add_comments_to_configuration_object_does_not_raise():
|
||||||
|
|
||||||
|
|
||||||
def test_add_comments_to_configuration_object_with_skip_first_does_not_raise():
|
def test_add_comments_to_configuration_object_with_skip_first_does_not_raise():
|
||||||
config = module.yaml.comments.CommentedMap([('foo', 33)])
|
config = module.ruamel.yaml.comments.CommentedMap([('foo', 33)])
|
||||||
schema = {'type': 'object', 'properties': {'foo': {'description': 'Foo'}}}
|
schema = {'type': 'object', 'properties': {'foo': {'description': 'Foo'}}}
|
||||||
|
|
||||||
module.add_comments_to_configuration_object(config, schema, skip_first=True)
|
module.add_comments_to_configuration_object(config, schema, skip_first=True)
|
||||||
|
@ -168,7 +172,7 @@ def test_add_comments_to_configuration_object_with_skip_first_does_not_raise():
|
||||||
|
|
||||||
def test_remove_commented_out_sentinel_keeps_other_comments():
|
def test_remove_commented_out_sentinel_keeps_other_comments():
|
||||||
field_name = 'foo'
|
field_name = 'foo'
|
||||||
config = module.yaml.comments.CommentedMap([(field_name, 33)])
|
config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)])
|
||||||
config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.\nCOMMENT_OUT')
|
config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.\nCOMMENT_OUT')
|
||||||
|
|
||||||
module.remove_commented_out_sentinel(config, field_name)
|
module.remove_commented_out_sentinel(config, field_name)
|
||||||
|
@ -180,7 +184,7 @@ def test_remove_commented_out_sentinel_keeps_other_comments():
|
||||||
|
|
||||||
def test_remove_commented_out_sentinel_without_sentinel_keeps_other_comments():
|
def test_remove_commented_out_sentinel_without_sentinel_keeps_other_comments():
|
||||||
field_name = 'foo'
|
field_name = 'foo'
|
||||||
config = module.yaml.comments.CommentedMap([(field_name, 33)])
|
config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)])
|
||||||
config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.')
|
config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.')
|
||||||
|
|
||||||
module.remove_commented_out_sentinel(config, field_name)
|
module.remove_commented_out_sentinel(config, field_name)
|
||||||
|
@ -192,7 +196,7 @@ def test_remove_commented_out_sentinel_without_sentinel_keeps_other_comments():
|
||||||
|
|
||||||
def test_remove_commented_out_sentinel_on_unknown_field_does_not_raise():
|
def test_remove_commented_out_sentinel_on_unknown_field_does_not_raise():
|
||||||
field_name = 'foo'
|
field_name = 'foo'
|
||||||
config = module.yaml.comments.CommentedMap([(field_name, 33)])
|
config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)])
|
||||||
config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.')
|
config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.')
|
||||||
|
|
||||||
module.remove_commented_out_sentinel(config, 'unknown')
|
module.remove_commented_out_sentinel(config, 'unknown')
|
||||||
|
@ -201,7 +205,9 @@ def test_remove_commented_out_sentinel_on_unknown_field_does_not_raise():
|
||||||
def test_generate_sample_configuration_does_not_raise():
|
def test_generate_sample_configuration_does_not_raise():
|
||||||
builtins = flexmock(sys.modules['builtins'])
|
builtins = flexmock(sys.modules['builtins'])
|
||||||
builtins.should_receive('open').with_args('schema.yaml').and_return('')
|
builtins.should_receive('open').with_args('schema.yaml').and_return('')
|
||||||
flexmock(module.yaml).should_receive('round_trip_load')
|
flexmock(module.ruamel.yaml).should_receive('YAML').and_return(
|
||||||
|
flexmock(load=lambda filename: {})
|
||||||
|
)
|
||||||
flexmock(module).should_receive('schema_to_sample_configuration')
|
flexmock(module).should_receive('schema_to_sample_configuration')
|
||||||
flexmock(module).should_receive('merge_source_configuration_into_destination')
|
flexmock(module).should_receive('merge_source_configuration_into_destination')
|
||||||
flexmock(module).should_receive('render_configuration')
|
flexmock(module).should_receive('render_configuration')
|
||||||
|
@ -214,7 +220,9 @@ def test_generate_sample_configuration_does_not_raise():
|
||||||
def test_generate_sample_configuration_with_source_filename_does_not_raise():
|
def test_generate_sample_configuration_with_source_filename_does_not_raise():
|
||||||
builtins = flexmock(sys.modules['builtins'])
|
builtins = flexmock(sys.modules['builtins'])
|
||||||
builtins.should_receive('open').with_args('schema.yaml').and_return('')
|
builtins.should_receive('open').with_args('schema.yaml').and_return('')
|
||||||
flexmock(module.yaml).should_receive('round_trip_load')
|
flexmock(module.ruamel.yaml).should_receive('YAML').and_return(
|
||||||
|
flexmock(load=lambda filename: {})
|
||||||
|
)
|
||||||
flexmock(module.load).should_receive('load_configuration')
|
flexmock(module.load).should_receive('load_configuration')
|
||||||
flexmock(module.normalize).should_receive('normalize')
|
flexmock(module.normalize).should_receive('normalize')
|
||||||
flexmock(module).should_receive('schema_to_sample_configuration')
|
flexmock(module).should_receive('schema_to_sample_configuration')
|
||||||
|
@ -229,7 +237,9 @@ def test_generate_sample_configuration_with_source_filename_does_not_raise():
|
||||||
def test_generate_sample_configuration_with_dry_run_does_not_write_file():
|
def test_generate_sample_configuration_with_dry_run_does_not_write_file():
|
||||||
builtins = flexmock(sys.modules['builtins'])
|
builtins = flexmock(sys.modules['builtins'])
|
||||||
builtins.should_receive('open').with_args('schema.yaml').and_return('')
|
builtins.should_receive('open').with_args('schema.yaml').and_return('')
|
||||||
flexmock(module.yaml).should_receive('round_trip_load')
|
flexmock(module.ruamel.yaml).should_receive('YAML').and_return(
|
||||||
|
flexmock(load=lambda filename: {})
|
||||||
|
)
|
||||||
flexmock(module).should_receive('schema_to_sample_configuration')
|
flexmock(module).should_receive('schema_to_sample_configuration')
|
||||||
flexmock(module).should_receive('merge_source_configuration_into_destination')
|
flexmock(module).should_receive('merge_source_configuration_into_destination')
|
||||||
flexmock(module).should_receive('render_configuration')
|
flexmock(module).should_receive('render_configuration')
|
||||||
|
|
|
@ -12,36 +12,10 @@ def test_load_configuration_parses_contents():
|
||||||
config_file = io.StringIO('key: value')
|
config_file = io.StringIO('key: value')
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
assert module.load_configuration('config.yaml') == {'key': 'value'}
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
|
assert module.load_configuration('config.yaml', config_paths) == {'key': 'value'}
|
||||||
def test_load_configuration_replaces_constants():
|
assert config_paths == {'config.yaml', 'other.yaml'}
|
||||||
builtins = flexmock(sys.modules['builtins'])
|
|
||||||
config_file = io.StringIO(
|
|
||||||
'''
|
|
||||||
constants:
|
|
||||||
key: value
|
|
||||||
key: {key}
|
|
||||||
'''
|
|
||||||
)
|
|
||||||
config_file.name = 'config.yaml'
|
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
|
||||||
assert module.load_configuration('config.yaml') == {'key': 'value'}
|
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_replaces_complex_constants():
|
|
||||||
builtins = flexmock(sys.modules['builtins'])
|
|
||||||
config_file = io.StringIO(
|
|
||||||
'''
|
|
||||||
constants:
|
|
||||||
key:
|
|
||||||
subkey: value
|
|
||||||
key: {key}
|
|
||||||
'''
|
|
||||||
)
|
|
||||||
config_file.name = 'config.yaml'
|
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
|
||||||
assert module.load_configuration('config.yaml') == {'key': {'subkey': 'value'}}
|
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_with_only_integer_value_does_not_raise():
|
def test_load_configuration_with_only_integer_value_does_not_raise():
|
||||||
|
@ -49,7 +23,10 @@ def test_load_configuration_with_only_integer_value_does_not_raise():
|
||||||
config_file = io.StringIO('33')
|
config_file = io.StringIO('33')
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
assert module.load_configuration('config.yaml') == 33
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
|
assert module.load_configuration('config.yaml', config_paths) == 33
|
||||||
|
assert config_paths == {'config.yaml', 'other.yaml'}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_inlines_include_relative_to_current_directory():
|
def test_load_configuration_inlines_include_relative_to_current_directory():
|
||||||
|
@ -63,8 +40,10 @@ def test_load_configuration_inlines_include_relative_to_current_directory():
|
||||||
config_file = io.StringIO('key: !include include.yaml')
|
config_file = io.StringIO('key: !include include.yaml')
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
assert module.load_configuration('config.yaml') == {'key': 'value'}
|
assert module.load_configuration('config.yaml', config_paths) == {'key': 'value'}
|
||||||
|
assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_inlines_include_relative_to_config_parent_directory():
|
def test_load_configuration_inlines_include_relative_to_config_parent_directory():
|
||||||
|
@ -85,8 +64,10 @@ def test_load_configuration_inlines_include_relative_to_config_parent_directory(
|
||||||
config_file = io.StringIO('key: !include include.yaml')
|
config_file = io.StringIO('key: !include include.yaml')
|
||||||
config_file.name = '/etc/config.yaml'
|
config_file.name = '/etc/config.yaml'
|
||||||
builtins.should_receive('open').with_args('/etc/config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('/etc/config.yaml').and_return(config_file)
|
||||||
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
assert module.load_configuration('/etc/config.yaml') == {'key': 'value'}
|
assert module.load_configuration('/etc/config.yaml', config_paths) == {'key': 'value'}
|
||||||
|
assert config_paths == {'/etc/config.yaml', '/etc/include.yaml', 'other.yaml'}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_raises_if_relative_include_does_not_exist():
|
def test_load_configuration_raises_if_relative_include_does_not_exist():
|
||||||
|
@ -99,9 +80,10 @@ def test_load_configuration_raises_if_relative_include_does_not_exist():
|
||||||
config_file = io.StringIO('key: !include include.yaml')
|
config_file = io.StringIO('key: !include include.yaml')
|
||||||
config_file.name = '/etc/config.yaml'
|
config_file.name = '/etc/config.yaml'
|
||||||
builtins.should_receive('open').with_args('/etc/config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('/etc/config.yaml').and_return(config_file)
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
with pytest.raises(FileNotFoundError):
|
with pytest.raises(FileNotFoundError):
|
||||||
module.load_configuration('/etc/config.yaml')
|
module.load_configuration('/etc/config.yaml', config_paths)
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_inlines_absolute_include():
|
def test_load_configuration_inlines_absolute_include():
|
||||||
|
@ -115,8 +97,10 @@ def test_load_configuration_inlines_absolute_include():
|
||||||
config_file = io.StringIO('key: !include /root/include.yaml')
|
config_file = io.StringIO('key: !include /root/include.yaml')
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
assert module.load_configuration('config.yaml') == {'key': 'value'}
|
assert module.load_configuration('config.yaml', config_paths) == {'key': 'value'}
|
||||||
|
assert config_paths == {'config.yaml', '/root/include.yaml', 'other.yaml'}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_raises_if_absolute_include_does_not_exist():
|
def test_load_configuration_raises_if_absolute_include_does_not_exist():
|
||||||
|
@ -127,9 +111,10 @@ def test_load_configuration_raises_if_absolute_include_does_not_exist():
|
||||||
config_file = io.StringIO('key: !include /root/include.yaml')
|
config_file = io.StringIO('key: !include /root/include.yaml')
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
with pytest.raises(FileNotFoundError):
|
with pytest.raises(FileNotFoundError):
|
||||||
assert module.load_configuration('config.yaml')
|
assert module.load_configuration('config.yaml', config_paths)
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_inlines_multiple_file_include_as_list():
|
def test_load_configuration_inlines_multiple_file_include_as_list():
|
||||||
|
@ -146,8 +131,15 @@ def test_load_configuration_inlines_multiple_file_include_as_list():
|
||||||
config_file = io.StringIO('key: !include [/root/include1.yaml, /root/include2.yaml]')
|
config_file = io.StringIO('key: !include [/root/include1.yaml, /root/include2.yaml]')
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
assert module.load_configuration('config.yaml') == {'key': ['value2', 'value1']}
|
assert module.load_configuration('config.yaml', config_paths) == {'key': ['value2', 'value1']}
|
||||||
|
assert config_paths == {
|
||||||
|
'config.yaml',
|
||||||
|
'/root/include1.yaml',
|
||||||
|
'/root/include2.yaml',
|
||||||
|
'other.yaml',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_include_with_unsupported_filename_type_raises():
|
def test_load_configuration_include_with_unsupported_filename_type_raises():
|
||||||
|
@ -158,9 +150,10 @@ def test_load_configuration_include_with_unsupported_filename_type_raises():
|
||||||
config_file = io.StringIO('key: !include {path: /root/include.yaml}')
|
config_file = io.StringIO('key: !include {path: /root/include.yaml}')
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.load_configuration('config.yaml')
|
module.load_configuration('config.yaml', config_paths)
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_merges_include():
|
def test_load_configuration_merges_include():
|
||||||
|
@ -184,8 +177,13 @@ def test_load_configuration_merges_include():
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
|
assert module.load_configuration('config.yaml', config_paths) == {
|
||||||
|
'foo': 'override',
|
||||||
|
'baz': 'quux',
|
||||||
|
}
|
||||||
|
assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_merges_multiple_file_include():
|
def test_load_configuration_merges_multiple_file_include():
|
||||||
|
@ -217,12 +215,14 @@ def test_load_configuration_merges_multiple_file_include():
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
assert module.load_configuration('config.yaml') == {
|
assert module.load_configuration('config.yaml', config_paths) == {
|
||||||
'foo': 'override',
|
'foo': 'override',
|
||||||
'baz': 'second',
|
'baz': 'second',
|
||||||
'original': 'yes',
|
'original': 'yes',
|
||||||
}
|
}
|
||||||
|
assert config_paths == {'config.yaml', '/tmp/include1.yaml', '/tmp/include2.yaml', 'other.yaml'}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values():
|
def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values():
|
||||||
|
@ -255,11 +255,13 @@ def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_value
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
assert module.load_configuration('config.yaml') == {
|
assert module.load_configuration('config.yaml', config_paths) == {
|
||||||
'stuff': {'foo': 'override'},
|
'stuff': {'foo': 'override'},
|
||||||
'other': {'a': 'override', 'c': 'd'},
|
'other': {'a': 'override', 'c': 'd'},
|
||||||
}
|
}
|
||||||
|
assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_with_retain_tag_but_without_merge_include_raises():
|
def test_load_configuration_with_retain_tag_but_without_merge_include_raises():
|
||||||
|
@ -285,9 +287,10 @@ def test_load_configuration_with_retain_tag_but_without_merge_include_raises():
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.load_configuration('config.yaml')
|
module.load_configuration('config.yaml', config_paths)
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_with_retain_tag_on_scalar_raises():
|
def test_load_configuration_with_retain_tag_on_scalar_raises():
|
||||||
|
@ -313,9 +316,10 @@ def test_load_configuration_with_retain_tag_on_scalar_raises():
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.load_configuration('config.yaml')
|
module.load_configuration('config.yaml', config_paths)
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values():
|
def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values():
|
||||||
|
@ -344,8 +348,10 @@ def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_val
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']}
|
assert module.load_configuration('config.yaml', config_paths) == {'stuff': ['a', 'c', 'x', 'y']}
|
||||||
|
assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise():
|
def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise():
|
||||||
|
@ -374,8 +380,12 @@ def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_do
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = {'other.yaml'}
|
||||||
|
|
||||||
assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']}
|
assert module.load_configuration('config.yaml', config_paths) == {
|
||||||
|
'stuff': ['a', 'b', 'c', 'x', 'y']
|
||||||
|
}
|
||||||
|
assert config_paths == {'config.yaml', '/tmp/include.yaml', 'other.yaml'}
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_with_omit_tag_on_non_list_item_raises():
|
def test_load_configuration_with_omit_tag_on_non_list_item_raises():
|
||||||
|
@ -403,9 +413,10 @@ def test_load_configuration_with_omit_tag_on_non_list_item_raises():
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.load_configuration('config.yaml')
|
module.load_configuration('config.yaml', config_paths)
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises():
|
def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises():
|
||||||
|
@ -432,9 +443,10 @@ def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises():
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.load_configuration('config.yaml')
|
module.load_configuration('config.yaml', config_paths)
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_with_omit_tag_but_without_merge_raises():
|
def test_load_configuration_with_omit_tag_but_without_merge_raises():
|
||||||
|
@ -462,9 +474,10 @@ def test_load_configuration_with_omit_tag_but_without_merge_raises():
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.load_configuration('config.yaml')
|
module.load_configuration('config.yaml', config_paths)
|
||||||
|
|
||||||
|
|
||||||
def test_load_configuration_does_not_merge_include_list():
|
def test_load_configuration_does_not_merge_include_list():
|
||||||
|
@ -489,9 +502,10 @@ def test_load_configuration_does_not_merge_include_list():
|
||||||
)
|
)
|
||||||
config_file.name = 'config.yaml'
|
config_file.name = 'config.yaml'
|
||||||
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
|
||||||
|
config_paths = set()
|
||||||
|
|
||||||
with pytest.raises(module.ruamel.yaml.error.YAMLError):
|
with pytest.raises(module.ruamel.yaml.error.YAMLError):
|
||||||
assert module.load_configuration('config.yaml')
|
assert module.load_configuration('config.yaml', config_paths)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
|
|
@ -4,19 +4,24 @@ from borgmatic.config import override as module
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
'value,expected_result',
|
'value,expected_result,option_type',
|
||||||
(
|
(
|
||||||
('thing', 'thing'),
|
('thing', 'thing', 'string'),
|
||||||
('33', 33),
|
('33', 33, 'integer'),
|
||||||
('33b', '33b'),
|
('33', '33', 'string'),
|
||||||
('true', True),
|
('33b', '33b', 'integer'),
|
||||||
('false', False),
|
('33b', '33b', 'string'),
|
||||||
('[foo]', ['foo']),
|
('true', True, 'boolean'),
|
||||||
('[foo, bar]', ['foo', 'bar']),
|
('false', False, 'boolean'),
|
||||||
|
('true', 'true', 'string'),
|
||||||
|
('[foo]', ['foo'], 'array'),
|
||||||
|
('[foo]', '[foo]', 'string'),
|
||||||
|
('[foo, bar]', ['foo', 'bar'], 'array'),
|
||||||
|
('[foo, bar]', '[foo, bar]', 'string'),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
def test_convert_value_type_coerces_values(value, expected_result):
|
def test_convert_value_type_coerces_values(value, expected_result, option_type):
|
||||||
assert module.convert_value_type(value) == expected_result
|
assert module.convert_value_type(value, option_type) == expected_result
|
||||||
|
|
||||||
|
|
||||||
def test_apply_overrides_updates_config():
|
def test_apply_overrides_updates_config():
|
||||||
|
@ -24,17 +29,28 @@ def test_apply_overrides_updates_config():
|
||||||
'section.key=value1',
|
'section.key=value1',
|
||||||
'other_section.thing=value2',
|
'other_section.thing=value2',
|
||||||
'section.nested.key=value3',
|
'section.nested.key=value3',
|
||||||
|
'location.no_longer_in_location=value4',
|
||||||
'new.foo=bar',
|
'new.foo=bar',
|
||||||
|
'new.mylist=[baz]',
|
||||||
|
'new.nonlist=[quux]',
|
||||||
]
|
]
|
||||||
config = {
|
config = {
|
||||||
'section': {'key': 'value', 'other': 'other_value'},
|
'section': {'key': 'value', 'other': 'other_value'},
|
||||||
'other_section': {'thing': 'thing_value'},
|
'other_section': {'thing': 'thing_value'},
|
||||||
|
'no_longer_in_location': 'because_location_is_deprecated',
|
||||||
|
}
|
||||||
|
schema = {
|
||||||
|
'properties': {
|
||||||
|
'new': {'properties': {'mylist': {'type': 'array'}, 'nonlist': {'type': 'string'}}}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.apply_overrides(config, raw_overrides)
|
module.apply_overrides(config, schema, raw_overrides)
|
||||||
|
|
||||||
assert config == {
|
assert config == {
|
||||||
'section': {'key': 'value1', 'other': 'other_value', 'nested': {'key': 'value3'}},
|
'section': {'key': 'value1', 'other': 'other_value', 'nested': {'key': 'value3'}},
|
||||||
'other_section': {'thing': 'value2'},
|
'other_section': {'thing': 'value2'},
|
||||||
'new': {'foo': 'bar'},
|
'new': {'foo': 'bar', 'mylist': ['baz'], 'nonlist': '[quux]'},
|
||||||
|
'location': {'no_longer_in_location': 'value4'},
|
||||||
|
'no_longer_in_location': 'value4',
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,9 @@
|
||||||
|
import pkgutil
|
||||||
|
|
||||||
|
import borgmatic.actions
|
||||||
|
import borgmatic.config.load
|
||||||
|
import borgmatic.config.validate
|
||||||
|
|
||||||
MAXIMUM_LINE_LENGTH = 80
|
MAXIMUM_LINE_LENGTH = 80
|
||||||
|
|
||||||
|
|
||||||
|
@ -6,3 +12,23 @@ def test_schema_line_length_stays_under_limit():
|
||||||
|
|
||||||
for line in schema_file.readlines():
|
for line in schema_file.readlines():
|
||||||
assert len(line.rstrip('\n')) <= MAXIMUM_LINE_LENGTH
|
assert len(line.rstrip('\n')) <= MAXIMUM_LINE_LENGTH
|
||||||
|
|
||||||
|
|
||||||
|
ACTIONS_MODULE_NAMES_TO_OMIT = {'arguments', 'export_key', 'json'}
|
||||||
|
ACTIONS_MODULE_NAMES_TO_ADD = {'key', 'umount'}
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_skip_actions_correspond_to_supported_actions():
|
||||||
|
'''
|
||||||
|
Ensure that the allowed actions in the schema's "skip_actions" option don't drift from
|
||||||
|
borgmatic's actual supported actions.
|
||||||
|
'''
|
||||||
|
schema = borgmatic.config.load.load_configuration(borgmatic.config.validate.schema_filename())
|
||||||
|
schema_skip_actions = set(schema['properties']['skip_actions']['items']['enum'])
|
||||||
|
supported_actions = {
|
||||||
|
module.name.replace('_', '-')
|
||||||
|
for module in pkgutil.iter_modules(borgmatic.actions.__path__)
|
||||||
|
if module.name not in ACTIONS_MODULE_NAMES_TO_OMIT
|
||||||
|
}.union(ACTIONS_MODULE_NAMES_TO_ADD)
|
||||||
|
|
||||||
|
assert schema_skip_actions == supported_actions
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import io
|
import io
|
||||||
|
import os
|
||||||
import string
|
import string
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
@ -57,7 +58,7 @@ def test_parse_configuration_transforms_file_into_mapping():
|
||||||
'''
|
'''
|
||||||
)
|
)
|
||||||
|
|
||||||
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
||||||
|
|
||||||
assert config == {
|
assert config == {
|
||||||
'source_directories': ['/home', '/etc'],
|
'source_directories': ['/home', '/etc'],
|
||||||
|
@ -67,6 +68,7 @@ def test_parse_configuration_transforms_file_into_mapping():
|
||||||
'keep_minutely': 60,
|
'keep_minutely': 60,
|
||||||
'checks': [{'name': 'repository'}, {'name': 'archives'}],
|
'checks': [{'name': 'repository'}, {'name': 'archives'}],
|
||||||
}
|
}
|
||||||
|
assert config_paths == {'/tmp/config.yaml'}
|
||||||
assert logs == []
|
assert logs == []
|
||||||
|
|
||||||
|
|
||||||
|
@ -83,12 +85,13 @@ def test_parse_configuration_passes_through_quoted_punctuation():
|
||||||
'''
|
'''
|
||||||
)
|
)
|
||||||
|
|
||||||
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
||||||
|
|
||||||
assert config == {
|
assert config == {
|
||||||
'source_directories': [f'/home/{string.punctuation}'],
|
'source_directories': [f'/home/{string.punctuation}'],
|
||||||
'repositories': [{'path': 'test.borg'}],
|
'repositories': [{'path': 'test.borg'}],
|
||||||
}
|
}
|
||||||
|
assert config_paths == {'/tmp/config.yaml'}
|
||||||
assert logs == []
|
assert logs == []
|
||||||
|
|
||||||
|
|
||||||
|
@ -140,7 +143,7 @@ def test_parse_configuration_inlines_include_inside_deprecated_section():
|
||||||
include_file.name = 'include.yaml'
|
include_file.name = 'include.yaml'
|
||||||
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
|
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
|
||||||
|
|
||||||
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
||||||
|
|
||||||
assert config == {
|
assert config == {
|
||||||
'source_directories': ['/home'],
|
'source_directories': ['/home'],
|
||||||
|
@ -148,6 +151,7 @@ def test_parse_configuration_inlines_include_inside_deprecated_section():
|
||||||
'keep_daily': 7,
|
'keep_daily': 7,
|
||||||
'keep_hourly': 24,
|
'keep_hourly': 24,
|
||||||
}
|
}
|
||||||
|
assert config_paths == {'/tmp/include.yaml', '/tmp/config.yaml'}
|
||||||
assert len(logs) == 1
|
assert len(logs) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@ -174,7 +178,7 @@ def test_parse_configuration_merges_include():
|
||||||
include_file.name = 'include.yaml'
|
include_file.name = 'include.yaml'
|
||||||
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
|
builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
|
||||||
|
|
||||||
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
||||||
|
|
||||||
assert config == {
|
assert config == {
|
||||||
'source_directories': ['/home'],
|
'source_directories': ['/home'],
|
||||||
|
@ -182,6 +186,7 @@ def test_parse_configuration_merges_include():
|
||||||
'keep_daily': 1,
|
'keep_daily': 1,
|
||||||
'keep_hourly': 24,
|
'keep_hourly': 24,
|
||||||
}
|
}
|
||||||
|
assert config_paths == {'/tmp/include.yaml', '/tmp/config.yaml'}
|
||||||
assert logs == []
|
assert logs == []
|
||||||
|
|
||||||
|
|
||||||
|
@ -193,6 +198,9 @@ def test_parse_configuration_raises_for_missing_config_file():
|
||||||
def test_parse_configuration_raises_for_missing_schema_file():
|
def test_parse_configuration_raises_for_missing_schema_file():
|
||||||
mock_config_and_schema('')
|
mock_config_and_schema('')
|
||||||
builtins = flexmock(sys.modules['builtins'])
|
builtins = flexmock(sys.modules['builtins'])
|
||||||
|
builtins.should_receive('open').with_args('/tmp/config.yaml').and_return(
|
||||||
|
io.StringIO('foo: bar')
|
||||||
|
)
|
||||||
builtins.should_receive('open').with_args('/tmp/schema.yaml').and_raise(FileNotFoundError)
|
builtins.should_receive('open').with_args('/tmp/schema.yaml').and_raise(FileNotFoundError)
|
||||||
|
|
||||||
with pytest.raises(FileNotFoundError):
|
with pytest.raises(FileNotFoundError):
|
||||||
|
@ -232,8 +240,8 @@ def test_parse_configuration_applies_overrides():
|
||||||
'''
|
'''
|
||||||
)
|
)
|
||||||
|
|
||||||
config, logs = module.parse_configuration(
|
config, config_paths, logs = module.parse_configuration(
|
||||||
'/tmp/config.yaml', '/tmp/schema.yaml', overrides=['location.local_path=borg2']
|
'/tmp/config.yaml', '/tmp/schema.yaml', overrides=['local_path=borg2']
|
||||||
)
|
)
|
||||||
|
|
||||||
assert config == {
|
assert config == {
|
||||||
|
@ -241,10 +249,11 @@ def test_parse_configuration_applies_overrides():
|
||||||
'repositories': [{'path': 'hostname.borg'}],
|
'repositories': [{'path': 'hostname.borg'}],
|
||||||
'local_path': 'borg2',
|
'local_path': 'borg2',
|
||||||
}
|
}
|
||||||
|
assert config_paths == {'/tmp/config.yaml'}
|
||||||
assert logs == []
|
assert logs == []
|
||||||
|
|
||||||
|
|
||||||
def test_parse_configuration_applies_normalization():
|
def test_parse_configuration_applies_normalization_after_environment_variable_interpolation():
|
||||||
mock_config_and_schema(
|
mock_config_and_schema(
|
||||||
'''
|
'''
|
||||||
location:
|
location:
|
||||||
|
@ -252,17 +261,19 @@ def test_parse_configuration_applies_normalization():
|
||||||
- /home
|
- /home
|
||||||
|
|
||||||
repositories:
|
repositories:
|
||||||
- path: hostname.borg
|
- ${NO_EXIST:-user@hostname:repo}
|
||||||
|
|
||||||
exclude_if_present: .nobackup
|
exclude_if_present: .nobackup
|
||||||
'''
|
'''
|
||||||
)
|
)
|
||||||
|
flexmock(os).should_receive('getenv').replace_with(lambda variable_name, default: default)
|
||||||
|
|
||||||
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
|
||||||
|
|
||||||
assert config == {
|
assert config == {
|
||||||
'source_directories': ['/home'],
|
'source_directories': ['/home'],
|
||||||
'repositories': [{'path': 'hostname.borg'}],
|
'repositories': [{'path': 'ssh://user@hostname/./repo'}],
|
||||||
'exclude_if_present': ['.nobackup'],
|
'exclude_if_present': ['.nobackup'],
|
||||||
}
|
}
|
||||||
|
assert config_paths == {'/tmp/config.yaml'}
|
||||||
assert logs
|
assert logs
|
||||||
|
|
|
@ -0,0 +1,28 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from flexmock import flexmock
|
||||||
|
|
||||||
|
from borgmatic.hooks import apprise as module
|
||||||
|
|
||||||
|
|
||||||
|
def test_destroy_monitor_removes_apprise_handler():
|
||||||
|
logger = logging.getLogger()
|
||||||
|
original_handlers = list(logger.handlers)
|
||||||
|
module.borgmatic.hooks.logs.add_handler(
|
||||||
|
module.borgmatic.hooks.logs.Forgetful_buffering_handler(
|
||||||
|
identifier=module.HANDLER_IDENTIFIER, byte_capacity=100, log_level=1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock())
|
||||||
|
|
||||||
|
assert logger.handlers == original_handlers
|
||||||
|
|
||||||
|
|
||||||
|
def test_destroy_monitor_without_apprise_handler_does_not_raise():
|
||||||
|
logger = logging.getLogger()
|
||||||
|
original_handlers = list(logger.handlers)
|
||||||
|
|
||||||
|
module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock())
|
||||||
|
|
||||||
|
assert logger.handlers == original_handlers
|
|
@ -8,7 +8,11 @@ from borgmatic.hooks import healthchecks as module
|
||||||
def test_destroy_monitor_removes_healthchecks_handler():
|
def test_destroy_monitor_removes_healthchecks_handler():
|
||||||
logger = logging.getLogger()
|
logger = logging.getLogger()
|
||||||
original_handlers = list(logger.handlers)
|
original_handlers = list(logger.handlers)
|
||||||
logger.addHandler(module.Forgetful_buffering_handler(byte_capacity=100, log_level=1))
|
module.borgmatic.hooks.logs.add_handler(
|
||||||
|
module.borgmatic.hooks.logs.Forgetful_buffering_handler(
|
||||||
|
identifier=module.HANDLER_IDENTIFIER, byte_capacity=100, log_level=1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock())
|
module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock())
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,89 @@
|
||||||
|
import logging
|
||||||
|
import platform
|
||||||
|
|
||||||
|
from flexmock import flexmock
|
||||||
|
|
||||||
|
from borgmatic.hooks import loki as module
|
||||||
|
|
||||||
|
|
||||||
|
def test_initialize_monitor_replaces_labels():
|
||||||
|
'''
|
||||||
|
Assert that label placeholders get replaced.
|
||||||
|
'''
|
||||||
|
hook_config = {
|
||||||
|
'url': 'http://localhost:3100/loki/api/v1/push',
|
||||||
|
'labels': {'hostname': '__hostname', 'config': '__config', 'config_full': '__config_path'},
|
||||||
|
}
|
||||||
|
config_filename = '/mock/path/test.yaml'
|
||||||
|
dry_run = True
|
||||||
|
module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
||||||
|
|
||||||
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
|
if isinstance(handler, module.Loki_log_handler):
|
||||||
|
assert handler.buffer.root['streams'][0]['stream']['hostname'] == platform.node()
|
||||||
|
assert handler.buffer.root['streams'][0]['stream']['config'] == 'test.yaml'
|
||||||
|
assert handler.buffer.root['streams'][0]['stream']['config_full'] == config_filename
|
||||||
|
return
|
||||||
|
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
def test_initialize_monitor_adds_log_handler():
|
||||||
|
'''
|
||||||
|
Assert that calling initialize_monitor adds our logger to the root logger.
|
||||||
|
'''
|
||||||
|
hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}}
|
||||||
|
module.initialize_monitor(
|
||||||
|
hook_config,
|
||||||
|
flexmock(),
|
||||||
|
config_filename='test.yaml',
|
||||||
|
monitoring_log_level=flexmock(),
|
||||||
|
dry_run=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
|
if isinstance(handler, module.Loki_log_handler):
|
||||||
|
return
|
||||||
|
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
def test_ping_monitor_adds_log_message():
|
||||||
|
'''
|
||||||
|
Assert that calling ping_monitor adds a message to our logger.
|
||||||
|
'''
|
||||||
|
hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}}
|
||||||
|
config_filename = 'test.yaml'
|
||||||
|
dry_run = True
|
||||||
|
module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
||||||
|
module.ping_monitor(
|
||||||
|
hook_config, flexmock(), config_filename, module.monitor.State.FINISH, flexmock(), dry_run
|
||||||
|
)
|
||||||
|
|
||||||
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
|
if isinstance(handler, module.Loki_log_handler):
|
||||||
|
assert any(
|
||||||
|
map(
|
||||||
|
lambda log: log
|
||||||
|
== f'{config_filename}: {module.MONITOR_STATE_TO_LOKI[module.monitor.State.FINISH]} backup',
|
||||||
|
map(lambda x: x[1], handler.buffer.root['streams'][0]['values']),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
def test_destroy_monitor_removes_log_handler():
|
||||||
|
'''
|
||||||
|
Assert that destroy_monitor removes the logger from the root logger.
|
||||||
|
'''
|
||||||
|
hook_config = {'url': 'http://localhost:3100/loki/api/v1/push', 'labels': {'app': 'borgmatic'}}
|
||||||
|
config_filename = 'test.yaml'
|
||||||
|
dry_run = True
|
||||||
|
module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
||||||
|
module.destroy_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run)
|
||||||
|
|
||||||
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
|
if isinstance(handler, module.Loki_log_handler):
|
||||||
|
assert False
|
|
@ -11,7 +11,7 @@ from borgmatic import execute as module
|
||||||
def test_log_outputs_logs_each_line_separately():
|
def test_log_outputs_logs_each_line_separately():
|
||||||
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
|
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
|
||||||
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
|
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
|
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS)
|
||||||
|
|
||||||
hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
|
hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
|
||||||
flexmock(module).should_receive('output_buffer_for_process').with_args(
|
flexmock(module).should_receive('output_buffer_for_process').with_args(
|
||||||
|
@ -28,13 +28,14 @@ def test_log_outputs_logs_each_line_separately():
|
||||||
exclude_stdouts=(),
|
exclude_stdouts=(),
|
||||||
output_log_level=logging.INFO,
|
output_log_level=logging.INFO,
|
||||||
borg_local_path='borg',
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_log_outputs_skips_logs_for_process_with_none_stdout():
|
def test_log_outputs_skips_logs_for_process_with_none_stdout():
|
||||||
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').never()
|
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').never()
|
||||||
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
|
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
|
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS)
|
||||||
|
|
||||||
hi_process = subprocess.Popen(['echo', 'hi'], stdout=None)
|
hi_process = subprocess.Popen(['echo', 'hi'], stdout=None)
|
||||||
flexmock(module).should_receive('output_buffer_for_process').with_args(
|
flexmock(module).should_receive('output_buffer_for_process').with_args(
|
||||||
|
@ -51,12 +52,13 @@ def test_log_outputs_skips_logs_for_process_with_none_stdout():
|
||||||
exclude_stdouts=(),
|
exclude_stdouts=(),
|
||||||
output_log_level=logging.INFO,
|
output_log_level=logging.INFO,
|
||||||
borg_local_path='borg',
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_log_outputs_returns_output_without_logging_for_output_log_level_none():
|
def test_log_outputs_returns_output_without_logging_for_output_log_level_none():
|
||||||
flexmock(module.logger).should_receive('log').never()
|
flexmock(module.logger).should_receive('log').never()
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
|
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS)
|
||||||
|
|
||||||
hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
|
hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
|
||||||
flexmock(module).should_receive('output_buffer_for_process').with_args(
|
flexmock(module).should_receive('output_buffer_for_process').with_args(
|
||||||
|
@ -73,6 +75,7 @@ def test_log_outputs_returns_output_without_logging_for_output_log_level_none():
|
||||||
exclude_stdouts=(),
|
exclude_stdouts=(),
|
||||||
output_log_level=None,
|
output_log_level=None,
|
||||||
borg_local_path='borg',
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert captured_outputs == {hi_process: 'hi', there_process: 'there'}
|
assert captured_outputs == {hi_process: 'hi', there_process: 'there'}
|
||||||
|
@ -80,7 +83,7 @@ def test_log_outputs_returns_output_without_logging_for_output_log_level_none():
|
||||||
|
|
||||||
def test_log_outputs_includes_error_output_in_exception():
|
def test_log_outputs_includes_error_output_in_exception():
|
||||||
flexmock(module.logger).should_receive('log')
|
flexmock(module.logger).should_receive('log')
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
|
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.ERROR)
|
||||||
flexmock(module).should_receive('command_for_process').and_return('grep')
|
flexmock(module).should_receive('command_for_process').and_return('grep')
|
||||||
|
|
||||||
process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
|
@ -88,7 +91,11 @@ def test_log_outputs_includes_error_output_in_exception():
|
||||||
|
|
||||||
with pytest.raises(subprocess.CalledProcessError) as error:
|
with pytest.raises(subprocess.CalledProcessError) as error:
|
||||||
module.log_outputs(
|
module.log_outputs(
|
||||||
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
|
(process,),
|
||||||
|
exclude_stdouts=(),
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert error.value.output
|
assert error.value.output
|
||||||
|
@ -100,7 +107,7 @@ def test_log_outputs_logs_multiline_error_output():
|
||||||
of a process' traceback.
|
of a process' traceback.
|
||||||
'''
|
'''
|
||||||
flexmock(module.logger).should_receive('log')
|
flexmock(module.logger).should_receive('log')
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
|
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.ERROR)
|
||||||
flexmock(module).should_receive('command_for_process').and_return('grep')
|
flexmock(module).should_receive('command_for_process').and_return('grep')
|
||||||
|
|
||||||
process = subprocess.Popen(
|
process = subprocess.Popen(
|
||||||
|
@ -111,13 +118,17 @@ def test_log_outputs_logs_multiline_error_output():
|
||||||
|
|
||||||
with pytest.raises(subprocess.CalledProcessError):
|
with pytest.raises(subprocess.CalledProcessError):
|
||||||
module.log_outputs(
|
module.log_outputs(
|
||||||
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
|
(process,),
|
||||||
|
exclude_stdouts=(),
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdout():
|
def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdout():
|
||||||
flexmock(module.logger).should_receive('log')
|
flexmock(module.logger).should_receive('log')
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
|
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.ERROR)
|
||||||
flexmock(module).should_receive('command_for_process').and_return('grep')
|
flexmock(module).should_receive('command_for_process').and_return('grep')
|
||||||
|
|
||||||
process = subprocess.Popen(['grep'], stdout=None)
|
process = subprocess.Popen(['grep'], stdout=None)
|
||||||
|
@ -125,30 +136,43 @@ def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdou
|
||||||
|
|
||||||
with pytest.raises(subprocess.CalledProcessError) as error:
|
with pytest.raises(subprocess.CalledProcessError) as error:
|
||||||
module.log_outputs(
|
module.log_outputs(
|
||||||
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
|
(process,),
|
||||||
|
exclude_stdouts=(),
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert error.value.returncode == 2
|
assert error.value.returncode == 2
|
||||||
assert not error.value.output
|
assert not error.value.output
|
||||||
|
|
||||||
|
|
||||||
def test_log_outputs_kills_other_processes_when_one_errors():
|
def test_log_outputs_kills_other_processes_and_raises_when_one_errors():
|
||||||
flexmock(module.logger).should_receive('log')
|
flexmock(module.logger).should_receive('log')
|
||||||
flexmock(module).should_receive('command_for_process').and_return('grep')
|
flexmock(module).should_receive('command_for_process').and_return('grep')
|
||||||
|
|
||||||
process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').with_args(
|
flexmock(module).should_receive('interpret_exit_code').with_args(
|
||||||
['grep'], None, 'borg'
|
['grep'],
|
||||||
).and_return(False)
|
None,
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').with_args(
|
'borg',
|
||||||
['grep'], 2, 'borg'
|
None,
|
||||||
).and_return(True)
|
).and_return(module.Exit_status.SUCCESS)
|
||||||
|
flexmock(module).should_receive('interpret_exit_code').with_args(
|
||||||
|
['grep'],
|
||||||
|
2,
|
||||||
|
'borg',
|
||||||
|
None,
|
||||||
|
).and_return(module.Exit_status.ERROR)
|
||||||
other_process = subprocess.Popen(
|
other_process = subprocess.Popen(
|
||||||
['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||||
)
|
)
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').with_args(
|
flexmock(module).should_receive('interpret_exit_code').with_args(
|
||||||
['sleep', '2'], None, 'borg'
|
['sleep', '2'],
|
||||||
).and_return(False)
|
None,
|
||||||
|
'borg',
|
||||||
|
None,
|
||||||
|
).and_return(module.Exit_status.SUCCESS)
|
||||||
flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
|
flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
|
||||||
process.stdout
|
process.stdout
|
||||||
)
|
)
|
||||||
|
@ -163,12 +187,56 @@ def test_log_outputs_kills_other_processes_when_one_errors():
|
||||||
exclude_stdouts=(),
|
exclude_stdouts=(),
|
||||||
output_log_level=logging.INFO,
|
output_log_level=logging.INFO,
|
||||||
borg_local_path='borg',
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert error.value.returncode == 2
|
assert error.value.returncode == 2
|
||||||
assert error.value.output
|
assert error.value.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_outputs_kills_other_processes_and_returns_when_one_exits_with_warning():
|
||||||
|
flexmock(module.logger).should_receive('log')
|
||||||
|
flexmock(module).should_receive('command_for_process').and_return('grep')
|
||||||
|
|
||||||
|
process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
|
flexmock(module).should_receive('interpret_exit_code').with_args(
|
||||||
|
['grep'],
|
||||||
|
None,
|
||||||
|
'borg',
|
||||||
|
None,
|
||||||
|
).and_return(module.Exit_status.SUCCESS)
|
||||||
|
flexmock(module).should_receive('interpret_exit_code').with_args(
|
||||||
|
['grep'],
|
||||||
|
2,
|
||||||
|
'borg',
|
||||||
|
None,
|
||||||
|
).and_return(module.Exit_status.WARNING)
|
||||||
|
other_process = subprocess.Popen(
|
||||||
|
['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||||
|
)
|
||||||
|
flexmock(module).should_receive('interpret_exit_code').with_args(
|
||||||
|
['sleep', '2'],
|
||||||
|
None,
|
||||||
|
'borg',
|
||||||
|
None,
|
||||||
|
).and_return(module.Exit_status.SUCCESS)
|
||||||
|
flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
|
||||||
|
process.stdout
|
||||||
|
)
|
||||||
|
flexmock(module).should_receive('output_buffer_for_process').with_args(
|
||||||
|
other_process, ()
|
||||||
|
).and_return(other_process.stdout)
|
||||||
|
flexmock(other_process).should_receive('kill').once()
|
||||||
|
|
||||||
|
module.log_outputs(
|
||||||
|
(process, other_process),
|
||||||
|
exclude_stdouts=(),
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_log_outputs_vents_other_processes_when_one_exits():
|
def test_log_outputs_vents_other_processes_when_one_exits():
|
||||||
'''
|
'''
|
||||||
Execute a command to generate a longish random string and pipe it into another command that
|
Execute a command to generate a longish random string and pipe it into another command that
|
||||||
|
@ -204,6 +272,7 @@ def test_log_outputs_vents_other_processes_when_one_exits():
|
||||||
exclude_stdouts=(process.stdout,),
|
exclude_stdouts=(process.stdout,),
|
||||||
output_log_level=logging.INFO,
|
output_log_level=logging.INFO,
|
||||||
borg_local_path='borg',
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -235,6 +304,7 @@ def test_log_outputs_does_not_error_when_one_process_exits():
|
||||||
exclude_stdouts=(process.stdout,),
|
exclude_stdouts=(process.stdout,),
|
||||||
output_log_level=logging.INFO,
|
output_log_level=logging.INFO,
|
||||||
borg_local_path='borg',
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -243,17 +313,27 @@ def test_log_outputs_truncates_long_error_output():
|
||||||
flexmock(module).should_receive('command_for_process').and_return('grep')
|
flexmock(module).should_receive('command_for_process').and_return('grep')
|
||||||
|
|
||||||
process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').with_args(
|
flexmock(module).should_receive('interpret_exit_code').with_args(
|
||||||
['grep'], None, 'borg'
|
['grep'],
|
||||||
).and_return(False)
|
None,
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').with_args(
|
'borg',
|
||||||
['grep'], 2, 'borg'
|
None,
|
||||||
).and_return(True)
|
).and_return(module.Exit_status.SUCCESS)
|
||||||
|
flexmock(module).should_receive('interpret_exit_code').with_args(
|
||||||
|
['grep'],
|
||||||
|
2,
|
||||||
|
'borg',
|
||||||
|
None,
|
||||||
|
).and_return(module.Exit_status.ERROR)
|
||||||
flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
|
flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
|
||||||
|
|
||||||
with pytest.raises(subprocess.CalledProcessError) as error:
|
with pytest.raises(subprocess.CalledProcessError) as error:
|
||||||
flexmock(module, ERROR_OUTPUT_MAX_LINE_COUNT=0).log_outputs(
|
flexmock(module, ERROR_OUTPUT_MAX_LINE_COUNT=0).log_outputs(
|
||||||
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
|
(process,),
|
||||||
|
exclude_stdouts=(),
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert error.value.returncode == 2
|
assert error.value.returncode == 2
|
||||||
|
@ -262,24 +342,32 @@ def test_log_outputs_truncates_long_error_output():
|
||||||
|
|
||||||
def test_log_outputs_with_no_output_logs_nothing():
|
def test_log_outputs_with_no_output_logs_nothing():
|
||||||
flexmock(module.logger).should_receive('log').never()
|
flexmock(module.logger).should_receive('log').never()
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
|
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS)
|
||||||
|
|
||||||
process = subprocess.Popen(['true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
process = subprocess.Popen(['true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
|
flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
|
||||||
|
|
||||||
module.log_outputs(
|
module.log_outputs(
|
||||||
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
|
(process,),
|
||||||
|
exclude_stdouts=(),
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_log_outputs_with_unfinished_process_re_polls():
|
def test_log_outputs_with_unfinished_process_re_polls():
|
||||||
flexmock(module.logger).should_receive('log').never()
|
flexmock(module.logger).should_receive('log').never()
|
||||||
flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
|
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS)
|
||||||
|
|
||||||
process = subprocess.Popen(['true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
process = subprocess.Popen(['true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
flexmock(process).should_receive('poll').and_return(None).and_return(0).times(3)
|
flexmock(process).should_receive('poll').and_return(None).and_return(0).times(3)
|
||||||
flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
|
flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
|
||||||
|
|
||||||
module.log_outputs(
|
module.log_outputs(
|
||||||
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
|
(process,),
|
||||||
|
exclude_stdouts=(),
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
borg_local_path='borg',
|
||||||
|
borg_exit_codes=None,
|
||||||
)
|
)
|
||||||
|
|
|
@ -9,6 +9,7 @@ def test_get_config_paths_returns_list_of_config_paths():
|
||||||
borgmatic_source_directory=None,
|
borgmatic_source_directory=None,
|
||||||
repository='repo',
|
repository='repo',
|
||||||
archive='archive',
|
archive='archive',
|
||||||
|
ssh_command=None,
|
||||||
)
|
)
|
||||||
global_arguments = flexmock(
|
global_arguments = flexmock(
|
||||||
dry_run=False,
|
dry_run=False,
|
||||||
|
@ -30,11 +31,46 @@ def test_get_config_paths_returns_list_of_config_paths():
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_config_paths_translates_ssh_command_argument_to_config():
|
||||||
|
bootstrap_arguments = flexmock(
|
||||||
|
borgmatic_source_directory=None,
|
||||||
|
repository='repo',
|
||||||
|
archive='archive',
|
||||||
|
ssh_command='ssh -i key',
|
||||||
|
)
|
||||||
|
global_arguments = flexmock(
|
||||||
|
dry_run=False,
|
||||||
|
)
|
||||||
|
local_borg_version = flexmock()
|
||||||
|
extract_process = flexmock(
|
||||||
|
stdout=flexmock(
|
||||||
|
read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
|
||||||
|
),
|
||||||
|
)
|
||||||
|
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args(
|
||||||
|
False,
|
||||||
|
'repo',
|
||||||
|
'archive',
|
||||||
|
object,
|
||||||
|
{'ssh_command': 'ssh -i key'},
|
||||||
|
object,
|
||||||
|
object,
|
||||||
|
extract_to_stdout=True,
|
||||||
|
).and_return(extract_process)
|
||||||
|
flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').with_args(
|
||||||
|
'repo', 'archive', {'ssh_command': 'ssh -i key'}, object, object
|
||||||
|
).and_return('archive')
|
||||||
|
assert module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) == [
|
||||||
|
'/borgmatic/config.yaml'
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_get_config_paths_with_missing_manifest_raises_value_error():
|
def test_get_config_paths_with_missing_manifest_raises_value_error():
|
||||||
bootstrap_arguments = flexmock(
|
bootstrap_arguments = flexmock(
|
||||||
borgmatic_source_directory=None,
|
borgmatic_source_directory=None,
|
||||||
repository='repo',
|
repository='repo',
|
||||||
archive='archive',
|
archive='archive',
|
||||||
|
ssh_command=None,
|
||||||
)
|
)
|
||||||
global_arguments = flexmock(
|
global_arguments = flexmock(
|
||||||
dry_run=False,
|
dry_run=False,
|
||||||
|
@ -57,6 +93,7 @@ def test_get_config_paths_with_broken_json_raises_value_error():
|
||||||
borgmatic_source_directory=None,
|
borgmatic_source_directory=None,
|
||||||
repository='repo',
|
repository='repo',
|
||||||
archive='archive',
|
archive='archive',
|
||||||
|
ssh_command=None,
|
||||||
)
|
)
|
||||||
global_arguments = flexmock(
|
global_arguments = flexmock(
|
||||||
dry_run=False,
|
dry_run=False,
|
||||||
|
@ -81,6 +118,7 @@ def test_get_config_paths_with_json_missing_key_raises_value_error():
|
||||||
borgmatic_source_directory=None,
|
borgmatic_source_directory=None,
|
||||||
repository='repo',
|
repository='repo',
|
||||||
archive='archive',
|
archive='archive',
|
||||||
|
ssh_command=None,
|
||||||
)
|
)
|
||||||
global_arguments = flexmock(
|
global_arguments = flexmock(
|
||||||
dry_run=False,
|
dry_run=False,
|
||||||
|
@ -101,6 +139,7 @@ def test_get_config_paths_with_json_missing_key_raises_value_error():
|
||||||
|
|
||||||
|
|
||||||
def test_run_bootstrap_does_not_raise():
|
def test_run_bootstrap_does_not_raise():
|
||||||
|
flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml'])
|
||||||
bootstrap_arguments = flexmock(
|
bootstrap_arguments = flexmock(
|
||||||
repository='repo',
|
repository='repo',
|
||||||
archive='archive',
|
archive='archive',
|
||||||
|
@ -108,6 +147,7 @@ def test_run_bootstrap_does_not_raise():
|
||||||
strip_components=1,
|
strip_components=1,
|
||||||
progress=False,
|
progress=False,
|
||||||
borgmatic_source_directory='/borgmatic',
|
borgmatic_source_directory='/borgmatic',
|
||||||
|
ssh_command=None,
|
||||||
)
|
)
|
||||||
global_arguments = flexmock(
|
global_arguments = flexmock(
|
||||||
dry_run=False,
|
dry_run=False,
|
||||||
|
@ -115,14 +155,54 @@ def test_run_bootstrap_does_not_raise():
|
||||||
local_borg_version = flexmock()
|
local_borg_version = flexmock()
|
||||||
extract_process = flexmock(
|
extract_process = flexmock(
|
||||||
stdout=flexmock(
|
stdout=flexmock(
|
||||||
read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
|
read=lambda: '{"config_paths": ["borgmatic/config.yaml"]}',
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
|
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
|
||||||
extract_process
|
extract_process
|
||||||
).twice()
|
).once()
|
||||||
flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
|
flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
|
||||||
'archive'
|
'archive'
|
||||||
)
|
)
|
||||||
|
|
||||||
module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version)
|
module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version)
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_bootstrap_translates_ssh_command_argument_to_config():
|
||||||
|
flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml'])
|
||||||
|
bootstrap_arguments = flexmock(
|
||||||
|
repository='repo',
|
||||||
|
archive='archive',
|
||||||
|
destination='dest',
|
||||||
|
strip_components=1,
|
||||||
|
progress=False,
|
||||||
|
borgmatic_source_directory='/borgmatic',
|
||||||
|
ssh_command='ssh -i key',
|
||||||
|
)
|
||||||
|
global_arguments = flexmock(
|
||||||
|
dry_run=False,
|
||||||
|
)
|
||||||
|
local_borg_version = flexmock()
|
||||||
|
extract_process = flexmock(
|
||||||
|
stdout=flexmock(
|
||||||
|
read=lambda: '{"config_paths": ["borgmatic/config.yaml"]}',
|
||||||
|
),
|
||||||
|
)
|
||||||
|
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args(
|
||||||
|
False,
|
||||||
|
'repo',
|
||||||
|
'archive',
|
||||||
|
object,
|
||||||
|
{'ssh_command': 'ssh -i key'},
|
||||||
|
object,
|
||||||
|
object,
|
||||||
|
extract_to_stdout=False,
|
||||||
|
destination_path='dest',
|
||||||
|
strip_components=1,
|
||||||
|
progress=False,
|
||||||
|
).and_return(extract_process).once()
|
||||||
|
flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').with_args(
|
||||||
|
'repo', 'archive', {'ssh_command': 'ssh -i key'}, object, object
|
||||||
|
).and_return('archive')
|
||||||
|
|
||||||
|
module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version)
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue