From 4d1959661667472790312fea9dcb9fa541b30620 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 22 Nov 2024 08:33:24 -0800 Subject: [PATCH] Add ZFS documentation (#261). --- NEWS | 1 + borgmatic/config/generate.py | 4 +- borgmatic/config/schema.yaml | 4 +- borgmatic/hooks/zfs.py | 4 +- docs/Dockerfile | 2 +- ...reparation-and-cleanup-steps-to-backups.md | 2 +- ...movable-drive-or-an-intermittent-server.md | 2 +- docs/how-to/customize-warnings-and-errors.md | 2 +- docs/how-to/develop-on-borgmatic.md | 2 +- docs/how-to/inspect-your-backups.md | 6 +- docs/how-to/run-arbitrary-borg-commands.md | 2 +- docs/how-to/snapshot-your-filesystems.md | 71 +++++++++++++++++++ docs/how-to/upgrade.md | 2 +- 13 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 docs/how-to/snapshot-your-filesystems.md diff --git a/NEWS b/NEWS index 2411f1c4..6cfeb561 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 1.9.3.dev0 + * #261: Add a ZFS hook for snapshotting ZFS datasets and backing them up. * Add a "--deleted" flag to the "repo-list" action for listing deleted archives that haven't yet been compacted (Borg 2 only). diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 85313c7e..c9f5aa81 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -44,12 +44,12 @@ def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): if example is not None: return example - if schema_type == 'array': + if schema_type == 'array' or 'array' in schema_type: config = ruamel.yaml.comments.CommentedSeq( [schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)] ) add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT)) - elif schema_type == 'object': + elif schema_type == 'object' or 'object' in schema_type: config = ruamel.yaml.comments.CommentedMap( [ (field_name, schema_to_sample_configuration(sub_schema, level + 1)) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 23a12f05..af604b43 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2255,7 +2255,7 @@ properties: config: "__config" hostname: "__hostname" description: | - Configuration for a monitoring integration with Grafana loki. You + Configuration for a monitoring integration with Grafana Loki. You can send the logs to a self-hosted instance or create an account at https://grafana.com/auth/sign-up/create-user. See borgmatic monitoring documentation for details. @@ -2280,4 +2280,4 @@ properties: Command to use instead of "umount". example: /usr/local/bin/umount description: | - Configuration for a integration with the ZFS filesystem. + Configuration for integration with the ZFS filesystem. diff --git a/borgmatic/hooks/zfs.py b/borgmatic/hooks/zfs.py index 717c75b9..34e6c4a5 100644 --- a/borgmatic/hooks/zfs.py +++ b/borgmatic/hooks/zfs.py @@ -284,12 +284,12 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ full_snapshot_names = get_all_snapshots(zfs_command) for full_snapshot_name in full_snapshot_names: - logger.debug(f'{log_prefix}: Destroying ZFS snapshot {full_snapshot_name}{dry_run_label}') - # Only destroy snapshots that borgmatic actually created! if not full_snapshot_name.split('@')[-1].startswith(BORGMATIC_SNAPSHOT_PREFIX): continue + logger.debug(f'{log_prefix}: Destroying ZFS snapshot {full_snapshot_name}{dry_run_label}') + if not dry_run: destroy_snapshot(zfs_command, full_snapshot_name) diff --git a/docs/Dockerfile b/docs/Dockerfile index fb78414e..c3279570 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -2,7 +2,7 @@ FROM docker.io/alpine:3.20.1 AS borgmatic COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib -RUN pip install --break-system-packages --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml +RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key change-passphrase" borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index f1f5da8e..02e41f43 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -3,7 +3,7 @@ title: How to add preparation and cleanup steps to backups eleventyNavigation: key: 🧹 Add preparation and cleanup steps parent: How-to guides - order: 9 + order: 10 --- ## Preparation and cleanup hooks diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index 31c987b0..d54d18f5 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -3,7 +3,7 @@ title: How to backup to a removable drive or an intermittent server eleventyNavigation: key: 💾 Backup to a removable drive/server parent: How-to guides - order: 10 + order: 11 --- ## Occasional backups diff --git a/docs/how-to/customize-warnings-and-errors.md b/docs/how-to/customize-warnings-and-errors.md index 36446fcc..74174658 100644 --- a/docs/how-to/customize-warnings-and-errors.md +++ b/docs/how-to/customize-warnings-and-errors.md @@ -3,7 +3,7 @@ title: How to customize warnings and errors eleventyNavigation: key: 💥 Customize warnings/errors parent: How-to guides - order: 12 + order: 13 --- ## When things go wrong diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index 21889543..bfacd8e5 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -3,7 +3,7 @@ title: How to develop on borgmatic eleventyNavigation: key: 🏗️ Develop on borgmatic parent: How-to guides - order: 14 + order: 15 --- ## Source code diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index e28dc807..1e75ca4e 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -119,10 +119,10 @@ archive, regardless of the user who performs the backup. (Note that Borg doesn't store the leading `/`.) With Borg version 1.2 and -earlierDatabase dump files are stored at a path dependent on the -[runtime +earlierDatabase dump files are stored at a path dependent on the [runtime directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory) -in use at the time the archive was created. +in use at the time the archive was created, as Borg 1.2 and earlier do not +support path rewriting. Prior to borgmatic version 1.9.0Database dump files were instead stored at `~/.borgmatic` within diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index dd231112..b9601e1b 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -3,7 +3,7 @@ title: How to run arbitrary Borg commands eleventyNavigation: key: 🔧 Run arbitrary Borg commands parent: How-to guides - order: 11 + order: 12 --- ## Running Borg with borgmatic diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md new file mode 100644 index 00000000..3aa61eed --- /dev/null +++ b/docs/how-to/snapshot-your-filesystems.md @@ -0,0 +1,71 @@ +--- +title: How to snapshot your filesystems +eleventyNavigation: + key: 📸 Snapshot your filesystems + parent: How-to guides + order: 9 +--- +## Filesystem hooks + +Many filesystems support taking snapshots—point-in-time, read-only "copies" of +your data, ideal for backing up files that may be changing during the backup. +These snapshots initially don't use any additional storage space and can be made +almost instantly. + + +### ZFS + +borgmatic supports taking and backing up snapshots with the ZFS filesystem. +First, you need one or more mounted ZFS datasets. Then, enable ZFS within +borgmatic by adding the following line to your configuration file: + +```yaml +zfs: +``` + +No other options are necessary, but if desired you can override some of the +commands used by the ZFS hook. For instance: + +```yaml +zfs: + zfs_command: /usr/local/bin/zfs + mount_command: /usr/local/bin/mount + umount_command: /usr/local/bin/umount +``` + +#### Dataset discovery + +You have a couple of options for borgmatic to find and backup your ZFS datasets: + + * For any dataset you'd like backed up, add its mount point to borgmatic's + `source_directories`. + * Or set the borgmatic-specific user property + `org.torsion.borgmatic:backup=auto` onto your dataset, e.g. by running `zfs + set org.torsion.borgmatic:backup=auto datasetname`. Then borgmatic can find + and backup these datasets. + +If you have multiple borgmatic configuration files with ZFS enabled, and you'd +like particular datasets to be backed up only for particular configuration +files, use the `source_directories` option. + +During a backup, borgmatic automatically snapshots these discovered datasets, +temporary mounts the snapshots within its [runtime +directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory), +and includes the snapshotted files in the backup. Additionally, borgmatic +rewrites the paths so that they appear at their original dataset locations in a +Borg archive. For instance, if your dataset is mounted at `/mnt/dataset`, then +the snapshotted files will appear in an archive at `/mnt/dataset` as well. + +With Borg version 1.2 and +earlierSnapshotted files are stored at a path dependent on the [runtime +directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory) +in use at the time the archive was created, as Borg 1.2 and earlier do not +support path rewriting. + + +#### Extract a dataset + +Given that filesystem snapshots are stored in a Borg archive as normal files, +you can use the standard +[extract action](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) to +extract them. diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index 8d34d5d3..06f97b43 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -3,7 +3,7 @@ title: How to upgrade borgmatic and Borg eleventyNavigation: key: 📦 Upgrade borgmatic/Borg parent: How-to guides - order: 13 + order: 14 --- ## Upgrading borgmatic