Compare commits

..

No commits in common. "main" and "master" have entirely different histories.
main ... master

231 changed files with 7082 additions and 18162 deletions

View File

@ -1,97 +1,56 @@
---
kind: pipeline
name: python-3-8-alpine-3-13
services:
- name: postgresql
image: docker.io/postgres:13.1-alpine
image: postgres:13.1-alpine
environment:
POSTGRES_PASSWORD: test
POSTGRES_DB: test
- name: postgresql2
image: docker.io/postgres:13.1-alpine
- name: mysql
image: mariadb:10.5
environment:
POSTGRES_PASSWORD: test2
POSTGRES_DB: test
commands:
- docker-entrypoint.sh -p 5433
- name: mariadb
image: docker.io/mariadb:10.11.4
environment:
MARIADB_ROOT_PASSWORD: test
MARIADB_DATABASE: test
- name: mariadb2
image: docker.io/mariadb:10.11.4
environment:
MARIADB_ROOT_PASSWORD: test2
MARIADB_DATABASE: test
commands:
- docker-entrypoint.sh --port=3307
- name: not-actually-mysql
image: docker.io/mariadb:10.11.4
environment:
MARIADB_ROOT_PASSWORD: test
MARIADB_DATABASE: test
- name: not-actually-mysql2
image: docker.io/mariadb:10.11.4
environment:
MARIADB_ROOT_PASSWORD: test2
MARIADB_DATABASE: test
commands:
- docker-entrypoint.sh --port=3307
MYSQL_ROOT_PASSWORD: test
MYSQL_DATABASE: test
- name: mongodb
image: docker.io/mongo:5.0.5
image: mongo:5.0.5
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: test
- name: mongodb2
image: docker.io/mongo:5.0.5
environment:
MONGO_INITDB_ROOT_USERNAME: root2
MONGO_INITDB_ROOT_PASSWORD: test2
commands:
- docker-entrypoint.sh --port=27018
clone:
skip_verify: true
steps:
- name: build
image: docker.io/alpine:3.13
environment:
TEST_CONTAINER: true
image: alpine:3.13
pull: always
commands:
- scripts/run-full-tests
---
kind: pipeline
name: documentation
type: exec
platform:
os: linux
arch: amd64
clone:
skip_verify: true
steps:
- name: build
environment:
USERNAME:
image: plugins/docker
settings:
username:
from_secret: docker_username
PASSWORD:
password:
from_secret: docker_password
IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
commands:
- podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org
- podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
- podman push "$IMAGE_NAME"
registry: projects.torsion.org
repo: projects.torsion.org/borgmatic-collective/borgmatic
tags: docs
dockerfile: docs/Dockerfile
trigger:
repo:
- borgmatic-collective/borgmatic
branch:
- main
- master
event:
- push

View File

@ -1,5 +1,4 @@
const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
const codeClipboard = require("eleventy-plugin-code-clipboard");
const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language");
const navigationPlugin = require("@11ty/eleventy-navigation");
@ -7,7 +6,6 @@ module.exports = function(eleventyConfig) {
eleventyConfig.addPlugin(pluginSyntaxHighlight);
eleventyConfig.addPlugin(inclusiveLangPlugin);
eleventyConfig.addPlugin(navigationPlugin);
eleventyConfig.addPlugin(codeClipboard);
let markdownIt = require("markdown-it");
let markdownItAnchor = require("markdown-it-anchor");
@ -33,7 +31,6 @@ module.exports = function(eleventyConfig) {
markdownIt(markdownItOptions)
.use(markdownItAnchor, markdownItAnchorOptions)
.use(markdownItReplaceLink)
.use(codeClipboard.markdownItCopyButton)
);
eleventyConfig.addPassthroughCopy({"docs/static": "static"});

View File

@ -1 +0,0 @@
select = Q0

35
.gitea/issue_template.md Normal file
View File

@ -0,0 +1,35 @@
#### What I'm trying to do and why
#### Steps to reproduce (if a bug)
Include (sanitized) borgmatic configuration files if applicable.
#### Actual behavior (if a bug)
Include (sanitized) `--verbosity 2` output if applicable.
#### Expected behavior (if a bug)
#### Other notes / implementation ideas
#### Environment
**borgmatic version:** [version here]
Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
**borgmatic installation method:** [e.g., Debian package, Docker container, etc.]
**Borg version:** [version here]
Use `sudo borg --version`
**Python version:** [version here]
Use `python3 --version`
**Database version (if applicable):** [version here]
Use `psql --version` or `mysql --version` on client and server.
**operating system and version:** [OS here]

View File

@ -1,77 +0,0 @@
name: "Bug or question/support"
about: "For filing a bug or getting support"
body:
- type: textarea
id: problem
attributes:
label: What I'm trying to do and why
validations:
required: true
- type: textarea
id: repro_steps
attributes:
label: Steps to reproduce
description: Include (sanitized) borgmatic configuration files if applicable.
validations:
required: false
- type: textarea
id: actual_behavior
attributes:
label: Actual behavior
description: Include (sanitized) `--verbosity 2` output if applicable.
validations:
required: false
- type: textarea
id: expected_behavior
attributes:
label: Expected behavior
validations:
required: false
- type: textarea
id: notes
attributes:
label: Other notes / implementation ideas
validations:
required: false
- type: input
id: borgmatic_version
attributes:
label: borgmatic version
description: Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
validations:
required: false
- type: input
id: borgmatic_install_method
attributes:
label: borgmatic installation method
description: e.g., pip install, Debian package, container, etc.
validations:
required: false
- type: input
id: borg_version
attributes:
label: Borg version
description: Use `sudo borg --version`
validations:
required: false
- type: input
id: python_version
attributes:
label: Python version
description: Use `python3 --version`
validations:
required: false
- type: input
id: database_version
attributes:
label: Database version (if applicable)
description: Use `psql --version` / `mysql --version` / `mongodump --version` / `sqlite3 --version`
validations:
required: false
- type: input
id: operating_system_version
attributes:
label: Operating system and version
description: On Linux, use `cat /etc/os-release`
validations:
required: false

View File

@ -1 +0,0 @@
blank_issues_enabled: true

View File

@ -1,15 +0,0 @@
name: "Feature"
about: "For filing a feature request or idea"
body:
- type: textarea
id: request
attributes:
label: What I'd like to do and why
validations:
required: true
- type: textarea
id: notes
attributes:
label: Other notes / implementation ideas
validations:
required: false

232
NEWS
View File

@ -1,227 +1,3 @@
1.8.3.dev0
* #743: Add a monitoring hook for sending backup status and logs to to Grafana Loki. See the
documentation for more information:
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook
* #753: When "archive_name_format" is not set, filter archives using the default archive name
format.
* #754: Fix error handling to log command output as one record per line instead of truncating
too-long output and swallowing the end of some Borg error messages.
* #757: Update documentation so "sudo borgmatic" works for pipx borgmatic installations.
* Update documentation to recommend installing/upgrading borgmatic with pipx instead of pip. See the
documentation for more information:
https://torsion.org/borgmatic/docs/how-to/set-up-backups/#installation
https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-borgmatic
1.8.2
* #345: Add "key export" action to export a copy of the repository key for safekeeping in case
the original goes missing or gets damaged.
* #727: Add a MariaDB database hook that uses native MariaDB commands instead of the deprecated
MySQL ones. Be aware though that any existing backups made with the "mysql_databases:" hook are
only restorable with a "mysql_databases:" configuration.
* #738: Fix for potential data loss (data not getting restored) in which the database "restore"
action didn't actually restore anything and indicated success anyway.
* Remove the deprecated use of the MongoDB hook's "--db" flag for database restoration.
* Add source code reference documentation for getting oriented with the borgmatic code as a
developer: https://torsion.org/borgmatic/docs/reference/source-code/
1.8.1
* #326: Add documentation for restoring a database to an alternate host:
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-to-an-alternate-host
* #697: Add documentation for "bootstrap" action:
https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive
* #725: Add "store_config_files" option for disabling the automatic backup of configuration files
used by the "config bootstrap" action.
* #728: Fix for "prune" action error when using the "keep_exclude_tags" option.
* #730: Fix for Borg's interactive prompt on the "check --repair" action automatically getting
answered "NO" even when the "check_i_know_what_i_am_doing" option isn't set.
* #732: Include multiple configuration files with a single "!include". See the documentation for
more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#multiple-merge-includes
* #734: Omit "--glob-archives" or "--match-archives" Borg flag when its value would be "*" (meaning
all archives).
1.8.0
* #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting
repository/archive into the resulting Borg command-line, pass repository to Borg via an
environment variable and make archive available for explicit use in your commands. See the
documentation for more information:
https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/
* #719: Fix an error when running "borg key export" through borgmatic.
* #720: Fix an error when dumping a database and the "exclude_nodump" option is set.
* #724: Add "check_i_know_what_i_am_doing" option to bypass Borg confirmation prompt when running
"check --repair".
* When merging two configuration files, error gracefully if the two files do not adhere to the same
format.
* #721: Remove configuration sections ("location:", "storage:", "hooks:", etc.), while still
keeping deprecated support for them. Now, all options are at the same level, and you don't need
to worry about commenting/uncommenting section headers when you change an option (if you remove
your sections first).
* #721: BREAKING: The retention prefix and the consistency prefix can no longer have different
values (unless one is not set).
* #721: BREAKING: The storage umask and the hooks umask can no longer have different values (unless
one is not set).
* BREAKING: Flags like "--config" that previously took multiple values now need to be given once
per value, e.g. "--config first.yaml --config second.yaml" instead of "--config first.yaml
second.yaml". This prevents argument parsing errors on ambiguous commands.
* BREAKING: Remove the deprecated (and silently ignored) "--successful" flag on the "list" action,
as newer versions of Borg list successful (non-checkpoint) archives by default.
* All deprecated configuration option values now generate warning logs.
* Remove the deprecated (and non-functional) "--excludes" flag in favor of excludes within
configuration.
* Fix an error when logging too-long command output during error handling. Now, long command output
is truncated before logging.
1.7.15
* #326: Add configuration options and command-line flags for backing up a database from one
location while restoring it somewhere else.
* #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors.
* #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style
configuration.
* #529: Deprecate generate-borgmatic-config in favor of new "config generate" action.
* #529: Deprecate validate-borgmatic-config in favor of new "config validate" action.
* #697, #712, #716: Extract borgmatic configuration from backup via new "config bootstrap"
action—even when borgmatic has no configuration yet!
* #669: Add sample systemd user service for running borgmatic as a non-root user.
* #711, #713: Fix an error when "data" check time files are accessed without getting upgraded
first.
1.7.14
* #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file,
or monitoring), so not even errors are shown.
* #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist.
* #659: Add Borg 2 date-based matching flags to various actions for archive selection.
* #703: Fix an error when loading the configuration schema on Fedora Linux.
* #704: Fix "check" action error when repository and archive checks are configured but the archive
check gets skipped due to the configured frequency.
* #706: Fix "--archive latest" on "list" and "info" actions that only worked on the first of
multiple configured repositories.
1.7.13
* #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema"
flag. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas
* #678: Fix error from PostgreSQL when dumping a database with a "format" of "plain".
* #678: Fix PostgreSQL hook to support "psql_command" and "pg_restore_command" options containing
commands with arguments.
* #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break
database dumping.
* #680: Add support for logging each log line as a JSON object via global "--log-json" flag.
* #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories.
* #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to
update your development checkouts accordingly.
* #686: Add fish shell completion script so you can tab-complete on the borgmatic command-line. See
the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion
* #687: Fix borgmatic error when not finding the configuration schema for certain "pip install
--editable" development installs.
* #688: Fix archive checks being skipped even when particular archives haven't been checked
recently. This occurred when using multiple borgmatic configuration files with different
"archive_name_format"s, for instance.
* #691: Fix error in "borgmatic restore" action when the configured repository path is relative
instead of absolute.
* #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like
"--progress" still work.
1.7.12
* #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file.
See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
* #666, #670: Fix error when running the "info" action with the "--match-archives" or "--archive"
flags. Also fix the "--match-archives"/"--archive" flags to correctly override the
"match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions.
* #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix"
options set.
* #672: Selectively shallow merge certain mappings or sequences when including configuration files.
See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge
* #672: Selectively omit list values when including configuration files. See the documentation for
more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#list-merge
* #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag.
See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes
* Add optional support for running end-to-end tests and building documentation with rootless Podman
instead of Docker.
1.7.11
* #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives
get used for borgmatic actions that operate on multiple archives. Override this behavior with the
new "match_archives" option in the storage section. This change is "breaking" in that it silently
changes which archives get considered for "rlist", "prune", "check", etc. See the documentation
for more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming
* #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format"
auto-matching behavior and the "match_archives" option.
* #658: Add "--log-file-format" flag for customizing the log message format. See the documentation
for more information:
https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#logging-to-file
* #662: Fix regression in which the "check_repositories" option failed to match repositories.
* #663: Fix regression in which the "transfer" action produced a traceback.
* Add spellchecking of source code during test runs.
1.7.10
* #396: When a database command errors, display and log the error message instead of swallowing it.
* #501: Optionally error if a source directory does not exist via "source_directories_must_exist"
option in borgmatic's location configuration.
* #576: Add support for "file://" paths within "repositories" option.
* #612: Define and use custom constants in borgmatic configuration files. See the documentation for
more information:
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#constant-interpolation
* #618: Add support for BORG_FILES_CACHE_TTL environment variable via "borg_files_cache_ttl" option
in borgmatic's storage configuration.
* #623: Fix confusing message when an error occurs running actions for a configuration file.
* #635: Add optional repository labels so you can select a repository via "--repository yourlabel"
at the command-line. See the configuration reference for more information:
https://torsion.org/borgmatic/docs/reference/configuration/
* #649: Add documentation on backing up a database running in a container:
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
* #655: Fix error when databases are configured and a source directory doesn't exist.
* Add code style plugins to enforce use of Python f-strings and prevent single-letter variables.
To join in the pedantry, refresh your test environment with "tox --recreate".
* Rename scripts/run-full-dev-tests to scripts/run-end-to-end-dev-tests and make it run end-to-end
tests only. Continue using tox to run unit and integration tests.
1.7.9
* #295: Add a SQLite database dump/restore hook.
* #304: Change the default action order when no actions are specified on the command-line to:
"create", "prune", "compact", "check". If you'd like to retain the old ordering ("prune" and
"compact" first), then specify actions explicitly on the command-line.
* #304: Run any command-line actions in the order specified instead of using a fixed ordering.
* #564: Add "--repository" flag to all actions where it makes sense, so you can run borgmatic on
a single configured repository instead of all of them.
* #628: Add a Healthchecks "log" state to send borgmatic logs to Healthchecks without signalling
success or failure.
* #647: Add "--strip-components all" feature on the "extract" action to remove leading path
components of files you extract. Must be used with the "--path" flag.
* Add support for Python 3.11.
1.7.8
* #620: With the "create" action and the "--list" ("--files") flag, only show excluded files at
verbosity 2.
* #621: Add optional authentication to the ntfy monitoring hook.
* With the "create" action, only one of "--list" ("--files") and "--progress" flags can be used.
This lines up with the new behavior in Borg 2.0.0b5.
* Internally support new Borg 2.0.0b5 "--filter" status characters / item flags for the "create"
action.
* Fix the "create" action with the "--dry-run" flag querying for databases when a PostgreSQL/MySQL
"all" database is configured. Now, these queries are skipped due to the dry run.
* Add "--repository" flag to the "rcreate" action to optionally select one configured repository to
create.
* Add "--progress" flag to the "transfer" action, new in Borg 2.0.0b5.
* Add "checkpoint_volume" configuration option to creates checkpoints every specified number of
bytes during a long-running backup, new in Borg 2.0.0b5.
1.7.7
* #642: Add MySQL database hook "add_drop_database" configuration option to control whether dumped
MySQL databases get dropped right before restore.
* #643: Fix for potential data loss (data not getting backed up) when dumping large "directory"
format PostgreSQL/MongoDB databases. Prior to the fix, these dumps would not finish writing to
disk before Borg consumed them. Now, the dumping process completes before Borg starts. This only
applies to "directory" format databases; other formats still stream to Borg without using
temporary disk space.
* Fix MongoDB "directory" format to work with mongodump/mongorestore without error. Prior to this
fix, only the "archive" format worked.
1.7.6
* #393, #438, #560: Optionally dump "all" PostgreSQL/MySQL databases to separate files instead of
one combined dump file, allowing more convenient restores of individual databases. You can enable
@ -486,7 +262,7 @@
* #398: Clarify canonical home of borgmatic in documentation.
* #406: Clarify that spaces in path names should not be backslashed in path names.
* #423: Fix error handling to error loudly when Borg gets killed due to running out of memory!
* Fix build so as not to attempt to build and push documentation for a non-main branch.
* Fix build so as not to attempt to build and push documentation for a non-master branch.
* "Fix" build failure with Alpine Edge by switching from Edge to Alpine 3.13.
* Move #borgmatic IRC channel from Freenode to Libera Chat due to Freenode takeover drama.
IRC connection info: https://torsion.org/borgmatic/#issues
@ -549,7 +325,7 @@
configuration schema descriptions.
1.5.6
* #292: Allow before_backup and similar hooks to exit with a soft failure without altering the
* #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the
monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring
services with a "start" status until after before_* hooks finish. Failures in before_* hooks
still trigger a monitoring "fail" status.
@ -618,7 +394,7 @@
* For "list" and "info" actions, show repository names even at verbosity 0.
1.4.22
* #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON output.
* #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON ouput.
* After a backup of a database dump in directory format, properly remove the dump directory.
* In "borgmatic --help", don't expand $HOME in listing of default "--config" paths.
@ -990,7 +766,7 @@
* #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files,
editor swap files, etc.
* #81: Document user-defined hooks run before/after backup, or on error.
* Add code style guidelines to the documentation.
* Add code style guidelines to the documention.
1.2.0
* #61: Support for Borg --list option via borgmatic command-line to list all archives.

View File

@ -11,46 +11,54 @@ borgmatic is simple, configuration-driven backup software for servers and
workstations. Protect your files with client-side encryption. Backup your
databases too. Monitor it all with integrated third-party services.
The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic/</a>
The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic</a>.
Here's an example configuration file:
```yaml
# List of source directories to backup.
source_directories:
- /home
- /etc
location:
# List of source directories to backup.
source_directories:
- /home
- /etc
# Paths of local or remote repositories to backup to.
repositories:
- path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo
label: borgbase
- path: /var/lib/backups/local.borg
label: local
# Paths of local or remote repositories to backup to.
repositories:
- ssh://1234@usw-s001.rsync.net/./backups.borg
- ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo
- /var/lib/backups/local.borg
# Retention policy for how many backups to keep.
keep_daily: 7
keep_weekly: 4
keep_monthly: 6
retention:
# Retention policy for how many backups to keep.
keep_daily: 7
keep_weekly: 4
keep_monthly: 6
# List of checks to run to validate your backups.
checks:
- name: repository
- name: archives
frequency: 2 weeks
consistency:
# List of checks to run to validate your backups.
checks:
- name: repository
- name: archives
frequency: 2 weeks
# Custom preparation scripts to run.
before_backup:
- prepare-for-backup.sh
hooks:
# Custom preparation scripts to run.
before_backup:
- prepare-for-backup.sh
# Databases to dump and include in backups.
postgresql_databases:
- name: users
# Databases to dump and include in backups.
postgresql_databases:
- name: users
# Third-party services to notify you if backups aren't happening.
healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
# Third-party services to notify you if backups aren't happening.
healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
```
Want to see borgmatic in action? Check out the <a
href="https://asciinema.org/a/203761?autoplay=1" target="_blank">screencast</a>.
<a href="https://asciinema.org/a/203761?autoplay=1" target="_blank"><img src="https://asciinema.org/a/203761.png" width="480"></a>
borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
## Integrations
@ -59,7 +67,6 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
@ -73,8 +80,8 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
Your first step is to [install and configure
borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/).
For additional documentation, check out the links above (left panel on wide screens)
for <a href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
For additional documentation, check out the links above for <a
href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
reference guides</a>.
@ -82,15 +89,16 @@ reference guides</a>.
Need somewhere to store your encrypted off-site backups? The following hosting
providers include specific support for Borg/borgmatic—and fund borgmatic
development and hosting when you use these referral links to sign up:
development and hosting when you use these links to sign up. (These are
referral links, but without any tracking scripts or cookies.)
<ul>
<li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
<li class="referral"><a href="https://hetzner.cloud/?ref=v9dOJ98Ic9I8">Hetzner</a>: A "storage box" that includes support for Borg</li>
</ul>
Additionally, rsync.net has a compatible storage offering, but does not fund
borgmatic development or hosting.
Additionally, [rsync.net](https://www.rsync.net/products/borg.html) and
[Hetzner](https://www.hetzner.com/storage/storage-box) have compatible storage
offerings, but do not currently fund borgmatic development or hosting.
## Support and contributing
@ -111,7 +119,10 @@ issues.
### Social
Follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic).
Check out the [Borg subreddit](https://www.reddit.com/r/BorgBackup/) for
general Borg and borgmatic discussion and support.
Also follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic).
### Chat
@ -152,5 +163,5 @@ Also, please check out the [borgmatic development
how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
info on cloning source code, running tests, etc.
<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/main)</a>
<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/master)</a>

View File

@ -7,8 +7,8 @@ permalink: security-policy/index.html
While we want to hear about security vulnerabilities in all versions of
borgmatic, security fixes are only made to the most recently released version.
It's not practical for our small volunteer effort to maintain multiple release
branches and put out separate security patches for each.
It's simply not practical for our small volunteer effort to maintain multiple
release branches and put out separate security patches for each.
## Reporting a vulnerability

View File

@ -1,9 +0,0 @@
import argparse
def update_arguments(arguments, **updates):
'''
Given an argparse.Namespace instance of command-line arguments and one or more keyword argument
updates to perform, return a copy of the arguments with those updates applied.
'''
return argparse.Namespace(**dict(vars(arguments), **updates))

View File

@ -8,13 +8,7 @@ logger = logging.getLogger(__name__)
def run_borg(
repository,
config,
local_borg_version,
borg_arguments,
global_arguments,
local_path,
remote_path,
repository, storage, local_borg_version, borg_arguments, local_path, remote_path,
):
'''
Run the "borg" action for the given repository.
@ -22,21 +16,18 @@ def run_borg(
if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
repository, borg_arguments.repository
):
logger.info(
f'{repository.get("label", repository["path"])}: Running arbitrary Borg command'
)
logger.info('{}: Running arbitrary Borg command'.format(repository))
archive_name = borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
repository,
borg_arguments.archive,
config,
storage,
local_borg_version,
global_arguments,
local_path,
remote_path,
)
borgmatic.borg.borg.run_arbitrary_borg(
repository['path'],
config,
repository,
storage,
local_borg_version,
options=borg_arguments.options,
archive=archive_name,

View File

@ -7,13 +7,7 @@ logger = logging.getLogger(__name__)
def run_break_lock(
repository,
config,
local_borg_version,
break_lock_arguments,
global_arguments,
local_path,
remote_path,
repository, storage, local_borg_version, break_lock_arguments, local_path, remote_path,
):
'''
Run the "break-lock" action for the given repository.
@ -21,14 +15,7 @@ def run_break_lock(
if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
repository, break_lock_arguments.repository
):
logger.info(
f'{repository.get("label", repository["path"])}: Breaking repository and cache locks'
)
logger.info(f'{repository}: Breaking repository and cache locks')
borgmatic.borg.break_lock.break_lock(
repository['path'],
config,
local_borg_version,
global_arguments,
local_path=local_path,
remote_path=remote_path,
repository, storage, local_borg_version, local_path=local_path, remote_path=remote_path,
)

View File

@ -1,7 +1,6 @@
import logging
import borgmatic.borg.check
import borgmatic.config.validate
import borgmatic.hooks.command
logger = logging.getLogger(__name__)
@ -10,7 +9,10 @@ logger = logging.getLogger(__name__)
def run_check(
config_filename,
repository,
config,
location,
storage,
consistency,
hooks,
hook_context,
local_borg_version,
check_arguments,
@ -21,25 +23,21 @@ def run_check(
'''
Run the "check" action for the given repository.
'''
if check_arguments.repository and not borgmatic.config.validate.repositories_match(
repository, check_arguments.repository
):
return
borgmatic.hooks.command.execute_hook(
config.get('before_check'),
config.get('umask'),
hooks.get('before_check'),
hooks.get('umask'),
config_filename,
'pre-check',
global_arguments.dry_run,
**hook_context,
)
logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks')
logger.info('{}: Running consistency checks'.format(repository))
borgmatic.borg.check.check_archives(
repository['path'],
config,
repository,
location,
storage,
consistency,
local_borg_version,
global_arguments,
local_path=local_path,
remote_path=remote_path,
progress=check_arguments.progress,
@ -48,8 +46,8 @@ def run_check(
force=check_arguments.force,
)
borgmatic.hooks.command.execute_hook(
config.get('after_check'),
config.get('umask'),
hooks.get('after_check'),
hooks.get('umask'),
config_filename,
'post-check',
global_arguments.dry_run,

View File

@ -2,7 +2,6 @@ import logging
import borgmatic.borg.compact
import borgmatic.borg.feature
import borgmatic.config.validate
import borgmatic.hooks.command
logger = logging.getLogger(__name__)
@ -11,7 +10,9 @@ logger = logging.getLogger(__name__)
def run_compact(
config_filename,
repository,
config,
storage,
retention,
hooks,
hook_context,
local_borg_version,
compact_arguments,
@ -23,29 +24,21 @@ def run_compact(
'''
Run the "compact" action for the given repository.
'''
if compact_arguments.repository and not borgmatic.config.validate.repositories_match(
repository, compact_arguments.repository
):
return
borgmatic.hooks.command.execute_hook(
config.get('before_compact'),
config.get('umask'),
hooks.get('before_compact'),
hooks.get('umask'),
config_filename,
'pre-compact',
global_arguments.dry_run,
**hook_context,
)
if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
logger.info(
f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}'
)
logger.info('{}: Compacting segments{}'.format(repository, dry_run_label))
borgmatic.borg.compact.compact_segments(
global_arguments.dry_run,
repository['path'],
config,
repository,
storage,
local_borg_version,
global_arguments,
local_path=local_path,
remote_path=remote_path,
progress=compact_arguments.progress,
@ -53,12 +46,10 @@ def run_compact(
threshold=compact_arguments.threshold,
)
else: # pragma: nocover
logger.info(
f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)'
)
logger.info('{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository))
borgmatic.hooks.command.execute_hook(
config.get('after_compact'),
config.get('umask'),
hooks.get('after_compact'),
hooks.get('umask'),
config_filename,
'post-compact',
global_arguments.dry_run,

View File

@ -1,103 +0,0 @@
import json
import logging
import os
import borgmatic.borg.extract
import borgmatic.borg.rlist
import borgmatic.config.validate
import borgmatic.hooks.command
from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
logger = logging.getLogger(__name__)
def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
'''
Given:
The bootstrap arguments, which include the repository and archive name, borgmatic source directory,
destination directory, and whether to strip components.
The global arguments, which include the dry run flag
and the local borg version,
Return:
The config paths from the manifest.json file in the borgmatic source directory after extracting it from the
repository.
Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
expected configuration path data.
'''
borgmatic_source_directory = (
bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY
)
borgmatic_manifest_path = os.path.expanduser(
os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
)
extract_process = borgmatic.borg.extract.extract_archive(
global_arguments.dry_run,
bootstrap_arguments.repository,
borgmatic.borg.rlist.resolve_archive_name(
bootstrap_arguments.repository,
bootstrap_arguments.archive,
{},
local_borg_version,
global_arguments,
),
[borgmatic_manifest_path],
{},
local_borg_version,
global_arguments,
extract_to_stdout=True,
)
manifest_json = extract_process.stdout.read()
if not manifest_json:
raise ValueError(
'Cannot read configuration paths from archive due to missing bootstrap manifest'
)
try:
manifest_data = json.loads(manifest_json)
except json.JSONDecodeError as error:
raise ValueError(
f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}'
)
try:
return manifest_data['config_paths']
except KeyError:
raise ValueError(
'Cannot read configuration paths from archive due to invalid bootstrap manifest'
)
def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
'''
Run the "bootstrap" action for the given repository.
Raise ValueError if the bootstrap configuration could not be loaded.
Raise CalledProcessError or OSError if Borg could not be run.
'''
manifest_config_paths = get_config_paths(
bootstrap_arguments, global_arguments, local_borg_version
)
logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
borgmatic.borg.extract.extract_archive(
global_arguments.dry_run,
bootstrap_arguments.repository,
borgmatic.borg.rlist.resolve_archive_name(
bootstrap_arguments.repository,
bootstrap_arguments.archive,
{},
local_borg_version,
global_arguments,
),
[config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
{},
local_borg_version,
global_arguments,
extract_to_stdout=False,
destination_path=bootstrap_arguments.destination,
strip_components=bootstrap_arguments.strip_components,
progress=bootstrap_arguments.progress,
)

View File

@ -1,48 +0,0 @@
import logging
import borgmatic.config.generate
import borgmatic.config.validate
import borgmatic.logger
logger = logging.getLogger(__name__)
def run_generate(generate_arguments, global_arguments):
'''
Given the generate arguments and the global arguments, each as an argparse.Namespace instance,
run the "generate" action.
Raise FileExistsError if a file already exists at the destination path and the generate
arguments do not have overwrite set.
'''
borgmatic.logger.add_custom_log_levels()
dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
logger.answer(
f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
)
borgmatic.config.generate.generate_sample_configuration(
global_arguments.dry_run,
generate_arguments.source_filename,
generate_arguments.destination_filename,
borgmatic.config.validate.schema_filename(),
overwrite=generate_arguments.overwrite,
)
if generate_arguments.source_filename:
logger.answer(
f'''
Merged in the contents of configuration file at: {generate_arguments.source_filename}
To review the changes made, run:
diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
)
logger.answer(
'''
This includes all available configuration options with example values, the few
required options as indicated. Please edit the file to suit your needs.
If you ever need help: https://torsion.org/borgmatic/#issues'''
)

View File

@ -1,25 +0,0 @@
import logging
import borgmatic.config.generate
import borgmatic.logger
logger = logging.getLogger(__name__)
def run_validate(validate_arguments, configs):
'''
Given the validate arguments as an argparse.Namespace instance and a dict of configuration
filename to corresponding parsed configuration, run the "validate" action.
Most of the validation is actually performed implicitly by the standard borgmatic configuration
loading machinery prior to here, so this function mainly exists to support additional validate
flags like "--show".
'''
borgmatic.logger.add_custom_log_levels()
if validate_arguments.show:
for config_path, config in configs.items():
if len(configs) > 1:
logger.answer('---')
logger.answer(borgmatic.config.generate.render_configuration(config))

View File

@ -1,15 +1,7 @@
import json
import logging
import os
try:
import importlib_metadata
except ModuleNotFoundError: # pragma: nocover
import importlib.metadata as importlib_metadata
import borgmatic.borg.create
import borgmatic.borg.state
import borgmatic.config.validate
import borgmatic.hooks.command
import borgmatic.hooks.dispatch
import borgmatic.hooks.dump
@ -17,39 +9,12 @@ import borgmatic.hooks.dump
logger = logging.getLogger(__name__)
def create_borgmatic_manifest(config, config_paths, dry_run):
'''
Create a borgmatic manifest file to store the paths to the configuration files used to create
the archive.
'''
if dry_run:
return
borgmatic_source_directory = config.get(
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
)
borgmatic_manifest_path = os.path.expanduser(
os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
)
if not os.path.exists(borgmatic_manifest_path):
os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True)
with open(borgmatic_manifest_path, 'w') as config_list_file:
json.dump(
{
'borgmatic_version': importlib_metadata.version('borgmatic'),
'config_paths': config_paths,
},
config_list_file,
)
def run_create(
config_filename,
repository,
config,
location,
storage,
hooks,
hook_context,
local_borg_version,
create_arguments,
@ -63,46 +28,39 @@ def run_create(
If create_arguments.json is True, yield the JSON output from creating the archive.
'''
if create_arguments.repository and not borgmatic.config.validate.repositories_match(
repository, create_arguments.repository
):
return
borgmatic.hooks.command.execute_hook(
config.get('before_backup'),
config.get('umask'),
hooks.get('before_backup'),
hooks.get('umask'),
config_filename,
'pre-backup',
global_arguments.dry_run,
**hook_context,
)
logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}')
logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
'remove_data_source_dumps',
config,
repository['path'],
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
'remove_database_dumps',
hooks,
repository,
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
active_dumps = borgmatic.hooks.dispatch.call_hooks(
'dump_data_sources',
config,
repository['path'],
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
'dump_databases',
hooks,
repository,
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
if config.get('store_config_files', True):
create_borgmatic_manifest(
config, global_arguments.used_config_paths, global_arguments.dry_run
)
stream_processes = [process for processes in active_dumps.values() for process in processes]
json_output = borgmatic.borg.create.create_archive(
global_arguments.dry_run,
repository['path'],
config,
repository,
location,
storage,
local_borg_version,
global_arguments,
local_path=local_path,
remote_path=remote_path,
progress=create_arguments.progress,
@ -115,15 +73,16 @@ def run_create(
yield json.loads(json_output)
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
'remove_data_source_dumps',
config,
'remove_database_dumps',
hooks,
config_filename,
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
borgmatic.hooks.command.execute_hook(
config.get('after_backup'),
config.get('umask'),
hooks.get('after_backup'),
hooks.get('umask'),
config_filename,
'post-backup',
global_arguments.dry_run,

View File

@ -1,33 +0,0 @@
import logging
import borgmatic.borg.export_key
import borgmatic.config.validate
logger = logging.getLogger(__name__)
def run_export_key(
repository,
config,
local_borg_version,
export_arguments,
global_arguments,
local_path,
remote_path,
):
'''
Run the "key export" action for the given repository.
'''
if export_arguments.repository is None or borgmatic.config.validate.repositories_match(
repository, export_arguments.repository
):
logger.info(f'{repository.get("label", repository["path"])}: Exporting repository key')
borgmatic.borg.export_key.export_key(
repository['path'],
config,
local_borg_version,
export_arguments,
global_arguments,
local_path=local_path,
remote_path=remote_path,
)

View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def run_export_tar(
repository,
config,
storage,
local_borg_version,
export_tar_arguments,
global_arguments,
@ -23,25 +23,23 @@ def run_export_tar(
repository, export_tar_arguments.repository
):
logger.info(
f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file'
'{}: Exporting archive {} as tar file'.format(repository, export_tar_arguments.archive)
)
borgmatic.borg.export_tar.export_tar_archive(
global_arguments.dry_run,
repository['path'],
repository,
borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
repository,
export_tar_arguments.archive,
config,
storage,
local_borg_version,
global_arguments,
local_path,
remote_path,
),
export_tar_arguments.paths,
export_tar_arguments.destination,
config,
storage,
local_borg_version,
global_arguments,
local_path=local_path,
remote_path=remote_path,
tar_filter=export_tar_arguments.tar_filter,

View File

@ -11,7 +11,9 @@ logger = logging.getLogger(__name__)
def run_extract(
config_filename,
repository,
config,
location,
storage,
hooks,
hook_context,
local_borg_version,
extract_arguments,
@ -23,8 +25,8 @@ def run_extract(
Run the "extract" action for the given repository.
'''
borgmatic.hooks.command.execute_hook(
config.get('before_extract'),