From b5b5c1fafa4433095f1ccdbf1075a3ad0f341f57 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 28 Nov 2024 18:47:15 -0800 Subject: [PATCH 1/9] Initial work on a Btrfs hook (#251). --- NEWS | 3 + borgmatic/config/paths.py | 11 +- borgmatic/config/schema.yaml | 14 ++ borgmatic/hooks/data_source/btrfs.py | 283 +++++++++++++++++++++++++++ borgmatic/hooks/data_source/zfs.py | 4 +- 5 files changed, 308 insertions(+), 7 deletions(-) create mode 100644 borgmatic/hooks/data_source/btrfs.py diff --git a/NEWS b/NEWS index c9a55a51..e8f686ff 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,7 @@ 1.9.4.dev0 + * #251 (beta): Add a Btrfs hook for snapshotting and backing up Btrfs subvolumes. See the + documentation for more information: + https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/ * #926: Fix library error when running within a PyInstaller bundle. * Reorganize data source and monitoring hooks to make developing new hooks easier. diff --git a/borgmatic/config/paths.py b/borgmatic/config/paths.py index 5482efef..ecc64b3b 100644 --- a/borgmatic/config/paths.py +++ b/borgmatic/config/paths.py @@ -33,10 +33,11 @@ def get_borgmatic_source_directory(config): TEMPORARY_DIRECTORY_PREFIX = 'borgmatic-' -def replace_temporary_subdirectory_with_glob(path): +def replace_temporary_subdirectory_with_glob(path, temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX): ''' - Given an absolute temporary directory path, look for a subdirectory within it starting with the - temporary directory prefix and replace it with an appropriate glob. For instance, given: + Given an absolute temporary directory path and an optional temporary directory prefix, look for + a subdirectory within it starting with the temporary directory prefix (or a default) and replace + it with an appropriate glob. For instance, given: /tmp/borgmatic-aet8kn93/borgmatic @@ -50,8 +51,8 @@ def replace_temporary_subdirectory_with_glob(path): '/', *( ( - f'{TEMPORARY_DIRECTORY_PREFIX}*' - if subdirectory.startswith(TEMPORARY_DIRECTORY_PREFIX) + f'{temporary_directory_prefix}*' + if subdirectory.startswith(temporary_directory_prefix) else subdirectory ) for subdirectory in path.split(os.path.sep) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index ddf9aaeb..9ba69a13 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2288,3 +2288,17 @@ properties: example: /usr/local/bin/umount description: | Configuration for integration with the ZFS filesystem. + btrfs: + type: ["object", "null"] + additionalProperties: false + properties: + btrfs_command: + type: string + description: | + Command to use instead of "btrfs". + example: /usr/local/bin/btrfs + findmnt_command: + type: string + description: | + Command to use instead of "findmnt". + example: /usr/local/bin/findmnt diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py new file mode 100644 index 00000000..0f7f46de --- /dev/null +++ b/borgmatic/hooks/data_source/btrfs.py @@ -0,0 +1,283 @@ +import functools +import glob +import logging +import os +import shutil +import subprocess + +import borgmatic.config.paths +import borgmatic.execute + +logger = logging.getLogger(__name__) + + +def use_streaming(hook_config, config, log_prefix): # pragma: no cover + ''' + Return whether dump streaming is used for this hook. (Spoiler: It isn't.) + ''' + return False + + +def get_subvolumes(btrfs_command, findmnt_command, source_directories=None): + ''' + Given a Btrfs command to run and a sequence of configured source directories, find the + intersection between the current Btrfs filesystem and subvolume mount points and the configured + borgmatic source directories. The idea is that these are the requested subvolumes to snapshot. + + If the source directories is None, then return all subvolumes. + + Return the result as a sequence of matching subvolume mount points. + ''' + # Find all top-level Btrfs filesystem mount points. + findmnt_output = borgmatic.execute.execute_command_and_capture_output( + ( + findmnt_command, + '-nt', + 'btrfs', + ) + ) + + try: + filesystem_mount_points = tuple( + line.rstrip().split(' ')[0] + for line in findmnt_output.splitlines() + ) + except ValueError: + raise ValueError('Invalid {findmnt_command} output') + + source_directories_set = set(source_directories or ()) + subvolumes = [] + + # For each filesystem mount point, find its subvolumes and match them again the given source + # directories to find the subvolumes to backup. Also try to match the filesystem mount point + # itself. + for mount_point in filesystem_mount_points: + if source_directories is None or mount_point in source_directories_set: + subvolumes.append(mount_point) + + btrfs_output = borgmatic.execute.execute_command_and_capture_output( + ( + btrfs_command, + 'subvolume', + 'list', + mount_point, + ) + ) + + try: + subvolumes.extend( + subvolume_path + for line in btrfs_output.splitlines() + for subvolume_subpath in (line.rstrip().split(' ')[-1],) + for subvolume_path in (os.path.join(mount_point, subvolume_subpath),) + if source_directories is None or subvolume_path in source_directories_set + ) + except ValueError: + raise ValueError('Invalid {btrfs_command} subvolume list output') + + return tuple(subvolumes) + + +BORGMATIC_SNAPSHOT_PREFIX = '.borgmatic-snapshot-' + + +def make_snapshot_path(subvolume_path): # pragma: no cover + ''' + Given the path to a subvolume, make a corresponding snapshot path for it. + ''' + return os.path.join( + subvolume_path, + f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}', + '.', # Borg 1.4+ "slashdot" hack. + # Included so that the snapshot ends up in the Borg archive at the "original" subvolume + # path. + subvolume_path.lstrip(os.path.sep), + ) + + +def make_snapshot_exclude_path(subvolume_path): # pragma: no cover + ''' + Given the path to a subvolume, make a corresponding exclude path for its embedded snapshot path. + This is to work around a quirk of Btrfs: If you make a snapshot path as a child directory of a + subvolume, then the snapshot's own initial directory component shows up as an empty directory + within the snapshot itself. For instance, if you have a Btrfs subvolume at /mnt and make a + snapshot of it at: + + /mnt/.borgmatic-snapshot-1234/mnt + + ... then the snapshot itself will have an empty directory at: + + /mnt/.borgmatic-snapshot-1234/mnt/.borgmatic-snapshot-1234 + + So to prevent that from ending up in the Borg archive, this function produces its path for + exclusion. + ''' + snapshot_directory = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' + + return os.path.join( + subvolume_path, + snapshot_directory, + subvolume_path.lstrip(os.path.sep), + snapshot_directory, + ) + + +def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path): # pragma: no cover + ''' + Given a Btrfs command to run, the path to a subvolume, and the path for a snapshot, create a new + Btrfs snapshot of the subvolume. + ''' + os.makedirs(os.path.dirname(snapshot_path), mode=0o700, exist_ok=True) + + borgmatic.execute.execute_command( + ( + btrfs_command, + 'subvolume', + 'snapshot', + subvolume_path, + snapshot_path, + ), + output_log_level=logging.DEBUG, + ) + + +def dump_data_sources( + hook_config, + config, + log_prefix, + config_paths, + borgmatic_runtime_directory, + source_directories, + dry_run, +): + ''' + Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic + configuration file paths, the borgmatic runtime directory, the configured source directories, + and whether this is a dry run, auto-detect and snapshot any Btrfs subvolume mount points listed + in the given source directories. Also update those source directories, replacing subvolume mount + points with corresponding snapshot directories so they get stored in the Borg archive instead. + Use the log prefix in any log entries. + + Return an empty sequence, since there are no ongoing dump processes from this hook. + + If this is a dry run, then don't actually snapshot anything. + ''' + dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' + logger.info(f'{log_prefix}: Snapshotting Btrfs datasets{dry_run_label}') + + # Based on the configured source directories, determine Btrfs subvolumes to backup. + btrfs_command = hook_config.get('btrfs_command', 'btrfs') + findmnt_command = hook_config.get('findmnt_command', 'findmnt') + requested_subvolume_paths = get_subvolumes(btrfs_command, findmnt_command, source_directories) + + # Snapshot each subvolume, rewriting source directories to use their snapshot paths. + for subvolume_path in requested_subvolume_paths: + logger.debug(f'{log_prefix}: Creating Btrfs snapshot for {subvolume_path} subvolume') + + snapshot_path = make_snapshot_path(subvolume_path) + + if dry_run: + continue + + snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path) + + if subvolume_path in source_directories: + source_directories.remove(subvolume_path) + + source_directories.append(snapshot_path) + config.setdefault('exclude_patterns', []).append(make_snapshot_exclude_path(subvolume_path)) + + return [] + + +def delete_snapshot(btrfs_command, snapshot_path): # pragma: no cover + ''' + Given a Btrfs command to run and the name of a snapshot path, delete it. + ''' + borgmatic.execute.execute_command( + ( + btrfs_command, + 'subvolume', + 'delete', + snapshot_path, + ), + output_log_level=logging.DEBUG, + ) + + +def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_directory, dry_run): + ''' + Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic runtime + directory, and whether this is a dry run, delete any Btrfs snapshots created by borgmatic. Use + the log prefix in any log entries. If this is a dry run, then don't actually remove anything. + ''' + dry_run_label = ' (dry run; not actually removing anything)' if dry_run else '' + + btrfs_command = hook_config.get('btrfs_command', 'btrfs') + findmnt_command = hook_config.get('findmnt_command', 'findmnt') + + try: + all_subvolume_paths = get_subvolumes(btrfs_command, findmnt_command) + except FileNotFoundError as error: + logger.debug(f'{log_prefix}: Could not find "{error.filename}" command') + return + except subprocess.CalledProcessError as error: + logger.debug(f'{log_prefix}: {error}') + return + + for subvolume_path in all_subvolume_paths: + subvolume_snapshots_glob = borgmatic.config.paths.replace_temporary_subdirectory_with_glob( + os.path.normpath(make_snapshot_path(subvolume_path)), + temporary_directory_prefix=BORGMATIC_SNAPSHOT_PREFIX, + ) + + logger.debug( + f'{log_prefix}: Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}' + ) + + for snapshot_path in glob.glob(subvolume_snapshots_glob): + if not os.path.isdir(snapshot_path): + continue + + logger.debug(f'{log_prefix}: Deleting Btrfs snapshot {snapshot_path}{dry_run_label}') + + if dry_run: + continue + + try: + delete_snapshot(btrfs_command, snapshot_path) + except FileNotFoundError: + logger.debug(f'{log_prefix}: Could not find "{btrfs_command}" command') + return + except subprocess.CalledProcessError as error: + logger.debug(f'{log_prefix}: {error}') + return + + # Strip off the subvolume path from the end of the snapshot path and then delete the + # resulting directory. + shutil.rmtree(snapshot_path.rsplit(subvolume_path, 1)[0]) + + +def make_data_source_dump_patterns( + hook_config, config, log_prefix, borgmatic_runtime_directory, name=None +): # pragma: no cover + ''' + Restores aren't implemented, because stored files can be extracted directly with "extract". + ''' + return () + + +def restore_data_source_dump( + hook_config, + config, + log_prefix, + data_source, + dry_run, + extract_process, + connection_params, + borgmatic_runtime_directory, +): # pragma: no cover + ''' + Restores aren't implemented, because stored files can be extracted directly with "extract". + ''' + raise NotImplementedError() diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 6e000674..a9e44c9d 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -104,6 +104,7 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path): # p first). ''' os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True) + borgmatic.execute.execute_command( ( mount_command, @@ -131,8 +132,7 @@ def dump_data_sources( is a dry run, auto-detect and snapshot any ZFS dataset mount points listed in the given source directories and any dataset with a borgmatic-specific user property. Also update those source directories, replacing dataset mount points with corresponding snapshot directories so they get - stored in the Borg archive instead of the dataset mount points. Use the log prefix in any log - entries. + stored in the Borg archive instead. Use the log prefix in any log entries. Return an empty sequence, since there are no ongoing dump processes from this hook. From 3f901c0a52060329d37752abbfad33225a78b18a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 28 Nov 2024 20:32:12 -0800 Subject: [PATCH 2/9] Btrfs hook documentation (#251). --- docs/how-to/snapshot-your-filesystems.md | 67 ++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md index 0e49090a..d8f5327f 100644 --- a/docs/how-to/snapshot-your-filesystems.md +++ b/docs/how-to/snapshot-your-filesystems.md @@ -52,7 +52,7 @@ feedback](https://torsion.org/borgmatic/#issues) you have on this feature. You have a couple of options for borgmatic to find and backup your ZFS datasets: * For any dataset you'd like backed up, add its mount point to borgmatic's - `source_directories`. + `source_directories` option. * Or set the borgmatic-specific user property `org.torsion.borgmatic:backup=auto` onto your dataset, e.g. by running `zfs set org.torsion.borgmatic:backup=auto datasetname`. Then borgmatic can find @@ -62,11 +62,11 @@ If you have multiple borgmatic configuration files with ZFS enabled, and you'd like particular datasets to be backed up only for particular configuration files, use the `source_directories` option instead of the user property. -During a backup, borgmatic automatically snapshots these discovered datasets, -temporary mounts the snapshots within its [runtime +During a backup, borgmatic automatically snapshots these discovered datasets +(non-recursively), temporary mounts the snapshots within its [runtime directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory), -and includes the snapshotted files in the files sent to Borg. borgmatic is -also responsible for cleaning up (destroying) these snapshots after a backup +and includes the snapshotted files in the paths sent to Borg. borgmatic is also +responsible for cleaning up (destroying) these snapshots after a backup completes. Additionally, borgmatic rewrites the snapshot file paths so that they appear @@ -88,3 +88,60 @@ Filesystem snapshots are stored in a Borg archive as normal files, so you can use the standard [extract action](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) to extract them. + + +### Btrfs + +New in version 1.9.4 Beta feature borgmatic supports taking +snapshots with the [Btrfs filesystem](https://btrfs.readthedocs.io/) and sending +those snapshots to Borg for backup. + +To use this feature, first you need one or more Btrfs subvolumes on mounted +filesystems. Then, enable Btrfs within borgmatic by adding the following line to +your configuration file: + +```yaml +btrfs: +``` + +No other options are necessary to enable Btrfs support, but if desired you can +override some of the commands used by the Btrfs hook. For instance: + +```yaml +btrfs: + btrfs_command: /usr/local/bin/btrfs + findmnt_command: /usr/local/bin/findmnt +``` + +As long as the Btrfs hook is in beta, it may be subject to breaking changes +and/or may not work well for your use cases. But feel free to use it in +production if you're okay with these caveats, and please [provide any +feedback](https://torsion.org/borgmatic/#issues) you have on this feature. + + +#### Subvolume discovery + +For any subvolume you'd like backed up, add its path to borgmatic's +`source_directories` option. During a backup, borgmatic snapshots these +subvolumes (non-recursively) and includes the snapshotted files in the paths +sent to Borg. borgmatic is also responsible for cleaning up (deleting) these +snapshots after a backup completes. + +Additionally, borgmatic rewrites the snapshot file paths so that they appear at +their original subvolume locations in a Borg archive. For instance, if your +subvolume exists at `/mnt/subvolume`, then the snapshotted files will appear in +an archive at `/mnt/subvolume` as well. + +With Borg version 1.2 and +earlierSnapshotted files are instead stored at a path dependent on the +temporary snapshot directory in use at the time the archive was created, as Borg +1.2 and earlier do not support path rewriting. + + +#### Extract a subvolume + +Subvolume snapshots are stored in a Borg archive as normal files, so you can use +the standard [extract +action](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) to extract +them. From d4a02f73b5ce1a6262cb3752e1ed6bcd4e3c672f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 28 Nov 2024 22:18:44 -0800 Subject: [PATCH 3/9] Create Btrfs snapshots as read-only (#251). --- borgmatic/hooks/data_source/btrfs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 0f7f46de..169f3061 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -134,6 +134,7 @@ def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path): # pragma: btrfs_command, 'subvolume', 'snapshot', + '-r', # Read-only, subvolume_path, snapshot_path, ), From 84a05522771ad764db027821b5915fd387696517 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 29 Nov 2024 09:36:46 -0800 Subject: [PATCH 4/9] Improve Btrfs hook factoring/organization (#251). --- borgmatic/hooks/data_source/btrfs.py | 96 ++++++++++++++++------------ 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 169f3061..de9fe265 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -18,6 +18,52 @@ def use_streaming(hook_config, config, log_prefix): # pragma: no cover return False +def get_filesystem_mount_points(findmnt_command): + ''' + Given a findmnt command to run, get all top-level Btrfs filesystem mount points. + ''' + findmnt_output = borgmatic.execute.execute_command_and_capture_output( + ( + findmnt_command, + '-nt', + 'btrfs', + ) + ) + + try: + return tuple( + line.rstrip().split(' ')[0] + for line in findmnt_output.splitlines() + ) + except ValueError: + raise ValueError('Invalid {findmnt_command} output') + + +def get_subvolumes_for_filesystem(btrfs_command, filesystem_mount_point): + ''' + Given a Btrfs command to run and a Btrfs filesystem mount point, get the subvolumes for that + filesystem. + ''' + btrfs_output = borgmatic.execute.execute_command_and_capture_output( + ( + btrfs_command, + 'subvolume', + 'list', + filesystem_mount_point, + ) + ) + + try: + return tuple( + subvolume_path + for line in btrfs_output.splitlines() + for subvolume_subpath in (line.rstrip().split(' ')[-1],) + for subvolume_path in (os.path.join(filesystem_mount_point, subvolume_subpath),) + ) + except ValueError: + raise ValueError('Invalid {btrfs_command} subvolume list output') + + def get_subvolumes(btrfs_command, findmnt_command, source_directories=None): ''' Given a Btrfs command to run and a sequence of configured source directories, find the @@ -28,53 +74,22 @@ def get_subvolumes(btrfs_command, findmnt_command, source_directories=None): Return the result as a sequence of matching subvolume mount points. ''' - # Find all top-level Btrfs filesystem mount points. - findmnt_output = borgmatic.execute.execute_command_and_capture_output( - ( - findmnt_command, - '-nt', - 'btrfs', - ) - ) - - try: - filesystem_mount_points = tuple( - line.rstrip().split(' ')[0] - for line in findmnt_output.splitlines() - ) - except ValueError: - raise ValueError('Invalid {findmnt_command} output') - - source_directories_set = set(source_directories or ()) + source_directories_lookup = set(source_directories or ()) subvolumes = [] # For each filesystem mount point, find its subvolumes and match them again the given source # directories to find the subvolumes to backup. Also try to match the filesystem mount point - # itself. - for mount_point in filesystem_mount_points: - if source_directories is None or mount_point in source_directories_set: + # itself (since it's implicitly a subvolume). + for mount_point in get_filesystem_mount_points(findmnt_command): + if source_directories is None or mount_point in source_directories_lookup: subvolumes.append(mount_point) - btrfs_output = borgmatic.execute.execute_command_and_capture_output( - ( - btrfs_command, - 'subvolume', - 'list', - mount_point, - ) + subvolumes.extend( + subvolume_path + for subvolume_path in get_subvolumes_for_filesystem(btrfs_command, mount_point) + if source_directories is None or subvolume_path in source_directories_lookup ) - try: - subvolumes.extend( - subvolume_path - for line in btrfs_output.splitlines() - for subvolume_subpath in (line.rstrip().split(' ')[-1],) - for subvolume_path in (os.path.join(mount_point, subvolume_subpath),) - if source_directories is None or subvolume_path in source_directories_set - ) - except ValueError: - raise ValueError('Invalid {btrfs_command} subvolume list output') - return tuple(subvolumes) @@ -169,10 +184,9 @@ def dump_data_sources( # Based on the configured source directories, determine Btrfs subvolumes to backup. btrfs_command = hook_config.get('btrfs_command', 'btrfs') findmnt_command = hook_config.get('findmnt_command', 'findmnt') - requested_subvolume_paths = get_subvolumes(btrfs_command, findmnt_command, source_directories) # Snapshot each subvolume, rewriting source directories to use their snapshot paths. - for subvolume_path in requested_subvolume_paths: + for subvolume_path in get_subvolumes(btrfs_command, findmnt_command, source_directories): logger.debug(f'{log_prefix}: Creating Btrfs snapshot for {subvolume_path} subvolume') snapshot_path = make_snapshot_path(subvolume_path) From 5bcc7b60c87632fc5e3e2ded4402f1342444a18e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 30 Nov 2024 09:32:50 -0800 Subject: [PATCH 5/9] Tests for Btrfs (#251). --- borgmatic/config/paths.py | 4 +- borgmatic/hooks/data_source/btrfs.py | 23 +- tests/unit/hooks/data_source/test_btrfs.py | 602 +++++++++++++++++++++ tests/unit/hooks/data_source/test_zfs.py | 25 +- 4 files changed, 630 insertions(+), 24 deletions(-) create mode 100644 tests/unit/hooks/data_source/test_btrfs.py diff --git a/borgmatic/config/paths.py b/borgmatic/config/paths.py index ecc64b3b..e5786237 100644 --- a/borgmatic/config/paths.py +++ b/borgmatic/config/paths.py @@ -33,7 +33,9 @@ def get_borgmatic_source_directory(config): TEMPORARY_DIRECTORY_PREFIX = 'borgmatic-' -def replace_temporary_subdirectory_with_glob(path, temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX): +def replace_temporary_subdirectory_with_glob( + path, temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX +): ''' Given an absolute temporary directory path and an optional temporary directory prefix, look for a subdirectory within it starting with the temporary directory prefix (or a default) and replace diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index de9fe265..8c78620c 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -30,13 +30,7 @@ def get_filesystem_mount_points(findmnt_command): ) ) - try: - return tuple( - line.rstrip().split(' ')[0] - for line in findmnt_output.splitlines() - ) - except ValueError: - raise ValueError('Invalid {findmnt_command} output') + return tuple(line.rstrip().split(' ')[0] for line in findmnt_output.splitlines()) def get_subvolumes_for_filesystem(btrfs_command, filesystem_mount_point): @@ -53,15 +47,12 @@ def get_subvolumes_for_filesystem(btrfs_command, filesystem_mount_point): ) ) - try: - return tuple( - subvolume_path - for line in btrfs_output.splitlines() - for subvolume_subpath in (line.rstrip().split(' ')[-1],) - for subvolume_path in (os.path.join(filesystem_mount_point, subvolume_subpath),) - ) - except ValueError: - raise ValueError('Invalid {btrfs_command} subvolume list output') + return tuple( + subvolume_path + for line in btrfs_output.splitlines() + for subvolume_subpath in (line.rstrip().split(' ')[-1],) + for subvolume_path in (os.path.join(filesystem_mount_point, subvolume_subpath),) + ) def get_subvolumes(btrfs_command, findmnt_command, source_directories=None): diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py new file mode 100644 index 00000000..d9bc2b42 --- /dev/null +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -0,0 +1,602 @@ +import pytest +from flexmock import flexmock + +from borgmatic.hooks.data_source import btrfs as module + + +def test_get_filesystem_mount_points_parses_findmnt_output(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + '/mnt0 /dev/loop0 btrfs rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/\n' + '/mnt1 /dev/loop1 btrfs rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/\n' + ) + + assert module.get_filesystem_mount_points('findmnt') == ('/mnt0', '/mnt1') + + +def test_get_subvolumes_for_filesystem_parses_subvolume_list_output(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'ID 270 gen 107 top level 5 path subvol1\n' 'ID 272 gen 74 top level 5 path subvol2\n' + ) + + assert module.get_subvolumes_for_filesystem('btrfs', '/mnt') == ('/mnt/subvol1', '/mnt/subvol2') + + +def test_get_subvolumes_collects_subvolumes_matching_source_directories_from_all_filesystems(): + flexmock(module).should_receive('get_filesystem_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('get_subvolumes_for_filesystem').with_args( + 'btrfs', '/mnt1' + ).and_return(('/one', '/two')) + flexmock(module).should_receive('get_subvolumes_for_filesystem').with_args( + 'btrfs', '/mnt2' + ).and_return(('/three', '/four')) + + assert module.get_subvolumes( + 'btrfs', 'findmnt', source_directories=['/one', '/four', '/five', '/six', '/mnt2', '/mnt3'] + ) == ('/one', '/mnt2', '/four') + + +def test_get_subvolumes_without_source_directories_collects_all_subvolumes_from_all_filesystems(): + flexmock(module).should_receive('get_filesystem_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('get_subvolumes_for_filesystem').with_args( + 'btrfs', '/mnt1' + ).and_return(('/one', '/two')) + flexmock(module).should_receive('get_subvolumes_for_filesystem').with_args( + 'btrfs', '/mnt2' + ).and_return(('/three', '/four')) + + assert module.get_subvolumes('btrfs', 'findmnt') == ( + '/mnt1', + '/one', + '/two', + '/mnt2', + '/three', + '/four', + ) + + +def test_dump_data_sources_snapshots_each_subvolume_and_updates_source_directories(): + source_directories = ['/foo', '/mnt/subvol1'] + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1', '/mnt/subvol2')) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol2').and_return( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2' + ) + flexmock(module).should_receive('snapshot_subvolume').with_args( + 'btrfs', '/mnt/subvol1', '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ).once() + flexmock(module).should_receive('snapshot_subvolume').with_args( + 'btrfs', '/mnt/subvol2', '/mnt/subvol2/.borgmatic-1234/mnt/subvol2' + ).once() + flexmock(module).should_receive('make_snapshot_exclude_path').with_args( + '/mnt/subvol1' + ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234') + flexmock(module).should_receive('make_snapshot_exclude_path').with_args( + '/mnt/subvol2' + ).and_return('/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234') + + assert ( + module.dump_data_sources( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + source_directories=source_directories, + dry_run=False, + ) + == [] + ) + + assert source_directories == [ + '/foo', + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1', + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2', + ] + assert config == { + 'btrfs': {}, + 'exclude_patterns': [ + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234', + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234', + ], + } + + +def test_dump_data_sources_uses_custom_btrfs_command_in_commands(): + source_directories = ['/foo', '/mnt/subvol1'] + config = {'btrfs': {'btrfs_command': '/usr/local/bin/btrfs'}} + flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1',)) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ) + flexmock(module).should_receive('snapshot_subvolume').with_args( + '/usr/local/bin/btrfs', '/mnt/subvol1', '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ).once() + flexmock(module).should_receive('make_snapshot_exclude_path').with_args( + '/mnt/subvol1' + ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234') + + assert ( + module.dump_data_sources( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + source_directories=source_directories, + dry_run=False, + ) + == [] + ) + + assert source_directories == [ + '/foo', + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1', + ] + assert config == { + 'btrfs': { + 'btrfs_command': '/usr/local/bin/btrfs', + }, + 'exclude_patterns': [ + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234', + ], + } + + +def test_dump_data_sources_uses_custom_findmnt_command_in_commands(): + source_directories = ['/foo', '/mnt/subvol1'] + config = {'btrfs': {'findmnt_command': '/usr/local/bin/findmnt'}} + flexmock(module).should_receive('get_subvolumes').with_args( + 'btrfs', '/usr/local/bin/findmnt', source_directories + ).and_return(('/mnt/subvol1',)).once() + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ) + flexmock(module).should_receive('snapshot_subvolume').with_args( + 'btrfs', '/mnt/subvol1', '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ).once() + flexmock(module).should_receive('make_snapshot_exclude_path').with_args( + '/mnt/subvol1' + ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234') + + assert ( + module.dump_data_sources( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + source_directories=source_directories, + dry_run=False, + ) + == [] + ) + + assert source_directories == [ + '/foo', + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1', + ] + assert config == { + 'btrfs': { + 'findmnt_command': '/usr/local/bin/findmnt', + }, + 'exclude_patterns': [ + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234', + ], + } + + +def test_dump_data_sources_with_dry_run_skips_snapshot_and_source_directories_update(): + source_directories = ['/foo', '/mnt/subvol1'] + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1',)) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ) + flexmock(module).should_receive('snapshot_subvolume').never() + flexmock(module).should_receive('make_snapshot_exclude_path').never() + + assert ( + module.dump_data_sources( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + source_directories=source_directories, + dry_run=True, + ) + == [] + ) + + assert source_directories == ['/foo', '/mnt/subvol1'] + assert config == {'btrfs': {}} + + +def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_source_directories_update(): + source_directories = ['/foo', '/mnt/subvol1'] + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return(()) + flexmock(module).should_receive('make_snapshot_path').never() + flexmock(module).should_receive('snapshot_subvolume').never() + flexmock(module).should_receive('make_snapshot_exclude_path').never() + + assert ( + module.dump_data_sources( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + source_directories=source_directories, + dry_run=False, + ) + == [] + ) + + assert source_directories == ['/foo', '/mnt/subvol1'] + assert config == {'btrfs': {}} + + +def test_remove_data_source_dumps_deletes_snapshots(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1', '/mnt/subvol2')) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/./mnt/subvol1' + ) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol2').and_return( + '/mnt/subvol2/.borgmatic-1234/./mnt/subvol2' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol1/.borgmatic-*/mnt/subvol1' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol2/.borgmatic-*/mnt/subvol2' + ) + flexmock(module.glob).should_receive('glob').with_args( + '/mnt/subvol1/.borgmatic-*/mnt/subvol1' + ).and_return( + ('/mnt/subvol1/.borgmatic-1234/mnt/subvol1', '/mnt/subvol1/.borgmatic-5678/mnt/subvol1') + ) + flexmock(module.glob).should_receive('glob').with_args( + '/mnt/subvol2/.borgmatic-*/mnt/subvol2' + ).and_return( + ('/mnt/subvol2/.borgmatic-1234/mnt/subvol2', '/mnt/subvol2/.borgmatic-5678/mnt/subvol2') + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-5678/mnt/subvol1' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-5678/mnt/subvol2' + ).and_return(False) + flexmock(module).should_receive('delete_snapshot').with_args( + 'btrfs', '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ).once() + flexmock(module).should_receive('delete_snapshot').with_args( + 'btrfs', '/mnt/subvol1/.borgmatic-5678/mnt/subvol1' + ).once() + flexmock(module).should_receive('delete_snapshot').with_args( + 'btrfs', '/mnt/subvol2/.borgmatic-1234/mnt/subvol2' + ).once() + flexmock(module).should_receive('delete_snapshot').with_args( + 'btrfs', '/mnt/subvol2/.borgmatic-5678/mnt/subvol2' + ).never() + flexmock(module.shutil).should_receive('rmtree').with_args( + '/mnt/subvol1/.borgmatic-1234' + ).once() + flexmock(module.shutil).should_receive('rmtree').with_args( + '/mnt/subvol1/.borgmatic-5678' + ).once() + flexmock(module.shutil).should_receive('rmtree').with_args( + '/mnt/subvol2/.borgmatic-1234' + ).once() + flexmock(module.shutil).should_receive('rmtree').with_args( + '/mnt/subvol2/.borgmatic-5678' + ).never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_with_get_subvolumes_file_not_found_error_bails(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_raise(FileNotFoundError) + flexmock(module).should_receive('make_snapshot_path').never() + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).never() + flexmock(module).should_receive('delete_snapshot').never() + flexmock(module.shutil).should_receive('rmtree').never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_with_get_subvolumes_called_process_error_bails(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_raise( + module.subprocess.CalledProcessError(1, 'command', 'error') + ) + flexmock(module).should_receive('make_snapshot_path').never() + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).never() + flexmock(module).should_receive('delete_snapshot').never() + flexmock(module.shutil).should_receive('rmtree').never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_with_dry_run_skips_deletes(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1', '/mnt/subvol2')) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/./mnt/subvol1' + ) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol2').and_return( + '/mnt/subvol2/.borgmatic-1234/./mnt/subvol2' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol1/.borgmatic-*/mnt/subvol1' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol2/.borgmatic-*/mnt/subvol2' + ) + flexmock(module.glob).should_receive('glob').with_args( + '/mnt/subvol1/.borgmatic-*/mnt/subvol1' + ).and_return( + ('/mnt/subvol1/.borgmatic-1234/mnt/subvol1', '/mnt/subvol1/.borgmatic-5678/mnt/subvol1') + ) + flexmock(module.glob).should_receive('glob').with_args( + '/mnt/subvol2/.borgmatic-*/mnt/subvol2' + ).and_return( + ('/mnt/subvol2/.borgmatic-1234/mnt/subvol2', '/mnt/subvol2/.borgmatic-5678/mnt/subvol2') + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-5678/mnt/subvol1' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-5678/mnt/subvol2' + ).and_return(False) + flexmock(module).should_receive('delete_snapshot').never() + flexmock(module.shutil).should_receive('rmtree').never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + borgmatic_runtime_directory='/run/borgmatic', + dry_run=True, + ) + + +def test_remove_data_source_dumps_without_subvolumes_skips_deletes(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return(()) + flexmock(module).should_receive('make_snapshot_path').never() + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).never() + flexmock(module).should_receive('delete_snapshot').never() + flexmock(module.shutil).should_receive('rmtree').never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_without_snapshots_skips_deletes(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1', '/mnt/subvol2')) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/./mnt/subvol1' + ) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol2').and_return( + '/mnt/subvol2/.borgmatic-1234/./mnt/subvol2' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol1/.borgmatic-*/mnt/subvol1' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol2/.borgmatic-*/mnt/subvol2' + ) + flexmock(module.glob).should_receive('glob').and_return(()) + flexmock(module.os.path).should_receive('isdir').never() + flexmock(module).should_receive('delete_snapshot').never() + flexmock(module.shutil).should_receive('rmtree').never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_with_delete_snapshot_file_not_found_error_bails(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1', '/mnt/subvol2')) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/./mnt/subvol1' + ) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol2').and_return( + '/mnt/subvol2/.borgmatic-1234/./mnt/subvol2' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol1/.borgmatic-*/mnt/subvol1' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol2/.borgmatic-*/mnt/subvol2' + ) + flexmock(module.glob).should_receive('glob').with_args( + '/mnt/subvol1/.borgmatic-*/mnt/subvol1' + ).and_return( + ('/mnt/subvol1/.borgmatic-1234/mnt/subvol1', '/mnt/subvol1/.borgmatic-5678/mnt/subvol1') + ) + flexmock(module.glob).should_receive('glob').with_args( + '/mnt/subvol2/.borgmatic-*/mnt/subvol2' + ).and_return( + ('/mnt/subvol2/.borgmatic-1234/mnt/subvol2', '/mnt/subvol2/.borgmatic-5678/mnt/subvol2') + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-5678/mnt/subvol1' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-5678/mnt/subvol2' + ).and_return(False) + flexmock(module).should_receive('delete_snapshot').and_raise(FileNotFoundError) + flexmock(module.shutil).should_receive('rmtree').never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_with_delete_snapshot_called_process_error_bails(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1', '/mnt/subvol2')) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/./mnt/subvol1' + ) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol2').and_return( + '/mnt/subvol2/.borgmatic-1234/./mnt/subvol2' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol1/.borgmatic-*/mnt/subvol1' + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/mnt/subvol2/.borgmatic-*/mnt/subvol2' + ) + flexmock(module.glob).should_receive('glob').with_args( + '/mnt/subvol1/.borgmatic-*/mnt/subvol1' + ).and_return( + ('/mnt/subvol1/.borgmatic-1234/mnt/subvol1', '/mnt/subvol1/.borgmatic-5678/mnt/subvol1') + ) + flexmock(module.glob).should_receive('glob').with_args( + '/mnt/subvol2/.borgmatic-*/mnt/subvol2' + ).and_return( + ('/mnt/subvol2/.borgmatic-1234/mnt/subvol2', '/mnt/subvol2/.borgmatic-5678/mnt/subvol2') + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-5678/mnt/subvol1' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-5678/mnt/subvol2' + ).and_return(False) + flexmock(module).should_receive('delete_snapshot').and_raise( + module.subprocess.CalledProcessError(1, 'command', 'error') + ) + flexmock(module.shutil).should_receive('rmtree').never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 7f634dad..cbb8cc9d 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -1,12 +1,13 @@ import pytest from flexmock import flexmock -import borgmatic.execute from borgmatic.hooks.data_source import zfs as module def test_get_datasets_to_backup_filters_datasets_by_source_directories(): - flexmock(borgmatic.execute).should_receive('execute_command_and_capture_output').and_return( + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( 'dataset\t/dataset\t-\nother\t/other\t-', ) @@ -16,7 +17,9 @@ def test_get_datasets_to_backup_filters_datasets_by_source_directories(): def test_get_datasets_to_backup_filters_datasets_by_user_property(): - flexmock(borgmatic.execute).should_receive('execute_command_and_capture_output').and_return( + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( 'dataset\t/dataset\tauto\nother\t/other\t-', ) @@ -26,7 +29,9 @@ def test_get_datasets_to_backup_filters_datasets_by_user_property(): def test_get_datasets_to_backup_with_invalid_list_output_raises(): - flexmock(borgmatic.execute).should_receive('execute_command_and_capture_output').and_return( + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( 'dataset', ) @@ -35,7 +40,9 @@ def test_get_datasets_to_backup_with_invalid_list_output_raises(): def test_get_get_all_datasets_does_not_filter_datasets(): - flexmock(borgmatic.execute).should_receive('execute_command_and_capture_output').and_return( + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( 'dataset\t/dataset\nother\t/other', ) @@ -46,7 +53,9 @@ def test_get_get_all_datasets_does_not_filter_datasets(): def test_get_all_datasets_with_invalid_list_output_raises(): - flexmock(borgmatic.execute).should_receive('execute_command_and_capture_output').and_return( + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( 'dataset', ) @@ -155,7 +164,9 @@ def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_source def test_get_all_snapshots_parses_list_output(): - flexmock(borgmatic.execute).should_receive('execute_command_and_capture_output').and_return( + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( 'dataset1@borgmatic-1234\ndataset2@borgmatic-4567', ) From a9a091081788991436eb19e9746f16c487621475 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 30 Nov 2024 09:36:52 -0800 Subject: [PATCH 6/9] Add Btrfs logo to integrations docs (#251). --- README.md | 1 + docs/static/btrfs.png | Bin 0 -> 6254 bytes 2 files changed, 1 insertion(+) create mode 100644 docs/static/btrfs.png diff --git a/README.md b/README.md index 9fc79c7f..5861a9a7 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). MongoDB SQLite OpenZFS +Btrfs rclone Healthchecks Uptime Kuma diff --git a/docs/static/btrfs.png b/docs/static/btrfs.png new file mode 100644 index 0000000000000000000000000000000000000000..7c52a2f30b264c90f889216850b363edced70d6d GIT binary patch literal 6254 zcmeHLcQl+`w^yRaBziDNlo555!Dta7ItfNEA-V`Ndhbm1F1kcFqD4uF-fKjn_d)b7 zA{gB9zUzK#ee1pJuKWM}o^{sQ``KroviI}bd!1)TJl9YpCuJbT!NDO{R+87k!NCLK z;Na>L<6|XY-0@N@xn(1(E{lUx9Y=;SC%`IPK39W1#mfKl{Ko_TKRvLPE6Iqp{K{2J zLmLMd5C0YcArUbNDcNmuz@57klvLC-v~>6A85n^~%q;gGu(GjpfH)s=ar5x<@q+~f zg@hp@qGC{S2}vpGM>3CPpUBBSRZvt?R#8<`*Le0^Q%f7BqpPQHU}$7)V*0|&{H2AZ zm9>qno&76^*N#qbgtLpQo4bdnm$#3vpMOALP;f|SSUB=cL}XNSOl*8YVp4KS>f5yR zjLfX;oZNTs^YRM{i;7E1QDq;>D=Mq1Ki1UN)qncj(D}qRoSD# z_a(yRzZmfAli@HfRUhVNB#81#nL;sJl4O(@rLk*nQ4d21Y1N1UB*t>4vaL0Ad6Dns zb}dvssxe-VANuibJsWo!4b#cmNqb9_N2nu(zUQ6f!dj^9S|T?NVoh@4<}}O)=*%M2 zcq>KJ<{CWDlg%{N=RSx$t!|2{kSiw{T&h%$^Beo5Q=nwnH6qiMTWG16Ppgyh=jDR> zNSAe@ZjnRM(@$QDb^M_e2vrJprOckGmzKZ2}%{X=bNu zs;>c9R37yV{?R3ott0OBEQMq{CFr(@$r#S`QC=Zg7jOeDo=0)3ZrtzBcsMH0-Vy3#0c^4j zDvb9+7+_9Yes=EGbuZ_sLnUCZ8yDeQqC|SZodY_Vfo_<>f%#wi?3T|Syil|ku-7cK zQ_Z1T9oo!)g3rPJXA^Q{!tZgxHI}C$TogR_?fiYpm{vk&zkK((4OTW;9bsAO+z|E4 znXg89GGRH+8)k)ZPhvJpm-QBR2Q{qc%mT4qgFI4R16fAj;&j?NVO7T5%bL z5UAVL2h)JxG3%rQ&gw-U$)9TdpfzgWLST_bSSiQ)kFWk0O8!Tn{0*Rgpyyve`6sZ5 z;W$V_xG_GUOLpe+<*SvI7+cbF#~PXs3^7G*n2V@#D_G&-xk&2Kl#8)g%p zoS=JU)ud_o;PP9p64&5utm~*;Vm(MV$_o$MC8{YK1HZByos;udQrV9(3D$X{Gk<>kh`Zu9wHt80j_Z*pxQi%{Ded2~6IXepVs zI6om@$33L9uhGmXaM+WI0QxAdLArq+CTfT;98pwx9 z({7c*inm-fs?8k=>T5p;Yjv48uTXlfjFZ>Q+lR;Gx$zUvq(p2FDkHSaH0i&6FnTdd zuuNSUA5UFI_ra|4!~V+VD}9qL;r-oaV=c|I>-5!SvZ{7z8wbgNFLUl;B9S9);p5D( z4jY& z>tUnW0%j1!d$ru-4L{#jvHt#MZxj^A@*L9(^x9BJP(^I`6iK;W7$Si4j)&)gA2fp~ zGh`>1VqBy&i}UUc7jL2xk-bm!mWj_@t~%+SGZ0DUMpLHp4YT_{nRe`ri2WX9SlnK} zXtjU!kqiO((<@!EhX}+LiV7taBkkN~H*R28#>1b3ZC8w@!NV;Dv5CSEQU+VVVnB zEVCtR3cZ4Mm53KmstEk0;mi-O=s=jRu-GFqwoDJ6o%-!?7&5O^z1Wg7yuhLX_pSZcdhe8v+-u*1Hi+^ zQ613bsGv@xnjr0az52p%&sVYB(Q;x)e1Ow1c^KjkKM|l(y|4os%^2AEs*@|aGZ_!q zh9>U)^sl`7zc3kXf_(8H%O9*St^yq6xBlQUhMdW8wbYQ8iJgfzwSeJc9I;vW5U5ro=Mp|IJXGO=SO_|sXZ%*>omyyLRtgvx|IZ9 z&UspbQCs>30?18PRz&SC-U7~aQAn*sKCP94+fpj7$aBpO;UQ@e@*q7I>>+<2^)V=m z2Fis%Z||$jUMtzr^P!kf2#Dk4z~$ZLix?QNPfJ@$gY0NR&7 zAzn@n3^QLZe~%4Ea)V`D^~a%U%wky=`5KTE?c#IC2yv0`;N^zi6f(^B?V?9%V$W3D z4;lRv>^OmR;uwj2qTIpya zPOFuZT1s%f`!V!5wHYTiQa16bN%Y#iBmBgOUQFEAd%rZbUEtU*0g|t{YI)QAy4%r|H|(#2)`y(d0Jbnt-nz( z2$Yk`A5acb&1d%yp@3cjxOC)?v5rUDFq)AKpp zwcC|)*`c9H}e6LoDe@g+C`V8Ny1Ddf&>)ylw6sdP-=gYW1j z`5%PY{Z#B>*uPkn;qAa0l*BOpDK zkenUAjS}d^{Ik1;C2DWI@oAw_fL^yS1Vp+Ymk2&J;#3xh4+ZjCOd@W={D%Y`HX3Dn zV0hTr?326+=n!8*Y|`5JZOe*%$=}zRxEVSx4ZxEBzA5}0Q^L}FWmhNMfdgzie3dF3 zc?*i&Ah5F#YAxn3Q0)`g$%bIhuvc6yRB5gpWqWIOZ@z3D{;<^xmrS7!QE0-AZ zFHM;C$QWhHye?2(8@gK6K=hzLlqyV1>>B}q4=r$_%#}CAMh>kF35hXO`^3cqwAm+t4J?CqKkJmaCdNaFE5zW{Sy;G>SiKZnQAjGu!tOGkO5WMa@fMW+~tTJF;<Noodp@qIB@GOC%AsP4*XuM-kM>I(jd~&Ry~Z;~}R{<;iYb-8?osMwWN_=hn75 zG!yQ-$_Gj!LYeQ&1hA*(t=yNKGfGvF0nMzxa&AzKQ*AXqW-Bm`Y(>GTTX>*ZU#O6K z7S0vA!}k5f#J%I1?~bTw{lU-~AdAot@}D_Cg$S{pH%&UVfl$LxiW z#}X>3QCY&hd||1%>4y}!uApsnlh6rqUAew>p{n$_m3qPb)Z@I25Lvj|w_}@7Jy#Cx z@LD0YX|Bc8q4C3nZO2$9=?=j)0ez1*>Af;u{F1+ z44c0ds56nj zojU{o*TBKa9qnB}x|ZK3p0Ah9F#Z7(%QO-Lpc)nQ!CCo1Qt(9G2?QlKY5r5Eaw*|v z4YfWEwD@K@`h$X-K+AlpC5_M3P_1>U1{cubMMAT{)dIc^XA0q5t)|i zwZ+B4!dN9oyL=W$9(dkHUoCrAJ#9s2w3_rhh)FAf`6<}a(RGy6V8dGSFM z1c7SD+8rjnVT$zf4VWNTyc8CtmR8%0&MyHCO2zj8xtkq>27}yG8PBUX#Nso)+Ojtr z-`$VezI-2vHp{>~oO7clmt*AqWC==f%6Te$%AygTj{yvz&5$X?U>gtQdx{j55CoJOF`J47P*n-s!cqjSq4pe zC&KFT9g&t~PfK*HpVMa{gJtQER~BgUZ+!cVqlt#y?OoPUQooOH8EVx)r?^<-@~{OC zSPeAod^T*oz$)LMwon%#a9K%uhu5cbG&Q5!T%T8pv)p?&uxE@LP?X-61u2$h(8kU! z?TE`W(!IzRsf!FDq4knJS~D9`7KsY+TB|>3JCehP6k8*1u@lQgnqH+c@0RKS)qE0M zsQGKwC_ z5Wn8%?LNmS&^R8Txib6DFvY@AABH0Xh7Y|hti8iwVhL`hgp?A(`xXjlaRJ!--9MY# zl0d$4P^aeJJ8X~sVP>nf1W$%QxZu!%mp8rArtJ;s3j*DY9AM6*b^wGxfHSjG#b=== zb^97iCoRbZIXLXHpNMa(qqfDNY^DabQitH2UhpokS*qIWjP5LqkdV_T-cZr5LTet) ztvw!TOPHhsgFZuoEca~()iN5HoZkXVM;F-O)ao5kV-+zEB0Neo{;Iz8XGWx(=LD|w zWKKnoji=kJF|E)%tV^~Nb=vP8aog)PmzKfJ3~rD*Y|6(BP-ZGK7)*hpJQ+)2u{LwD z`7ix|d+^VX%rW>p_OX6bU#tq0GEP1B?_M|8?AUs&lnkZajZB5z-|0FsyfZlWRoqWS zpcqU&?wvV(RPi+%%1a~8WBTUtr}t{CLgK~Fd39|vd%|4p<~<}=hUfhx%V*Aqb9yvdQ+%a2;pFN=Dnt%k@AEZNfBWKW zsk4>Jyxi64*S>R8V*AWhTDp+`cO6dNNtf=X4mYUn6LXCd`bIjD^i)sjrb4XzR71Yv IiCOS}0eQDt-2eap literal 0 HcmV?d00001 From 136626958696847762d485ffda6fa65495e58742 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 30 Nov 2024 09:44:55 -0800 Subject: [PATCH 7/9] Add a couple of missing tests (#251). --- borgmatic/config/schema.yaml | 2 ++ borgmatic/hooks/data_source/btrfs.py | 1 - tests/unit/config/test_paths.py | 16 ++++++++++++++++ tests/unit/hooks/data_source/test_btrfs.py | 1 - 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 9ba69a13..c2e900ef 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2302,3 +2302,5 @@ properties: description: | Command to use instead of "findmnt". example: /usr/local/bin/findmnt + description: | + Configuration for integration with the Btrfs filesystem. diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 8c78620c..4ea270fa 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -1,4 +1,3 @@ -import functools import glob import logging import os diff --git a/tests/unit/config/test_paths.py b/tests/unit/config/test_paths.py index 294212ba..a63c86e1 100644 --- a/tests/unit/config/test_paths.py +++ b/tests/unit/config/test_paths.py @@ -45,6 +45,22 @@ def test_replace_temporary_subdirectory_with_glob_transforms_path(): ) +def test_replace_temporary_subdirectory_with_glob_passes_through_non_matching_path(): + assert ( + module.replace_temporary_subdirectory_with_glob('/tmp/foo-aet8kn93/borgmatic') + == '/tmp/foo-aet8kn93/borgmatic' + ) + + +def test_replace_temporary_subdirectory_with_glob_uses_custom_temporary_directory_prefix(): + assert ( + module.replace_temporary_subdirectory_with_glob( + '/tmp/.borgmatic-aet8kn93/borgmatic', temporary_directory_prefix='.borgmatic-' + ) + == '/tmp/.borgmatic-*/borgmatic' + ) + + def test_runtime_directory_uses_config_option(): flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path) flexmock(module.os).should_receive('makedirs') diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index d9bc2b42..6e4fbf7d 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -1,4 +1,3 @@ -import pytest from flexmock import flexmock from borgmatic.hooks.data_source import btrfs as module From 0978c669adf72f2beb4918de2651532304f0be4d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 30 Nov 2024 10:25:01 -0800 Subject: [PATCH 8/9] A little more Btrfs error handling (#251). --- borgmatic/hooks/data_source/btrfs.py | 2 + tests/unit/hooks/data_source/test_btrfs.py | 71 +++++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 4ea270fa..7ab0f12f 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -51,6 +51,8 @@ def get_subvolumes_for_filesystem(btrfs_command, filesystem_mount_point): for line in btrfs_output.splitlines() for subvolume_subpath in (line.rstrip().split(' ')[-1],) for subvolume_path in (os.path.join(filesystem_mount_point, subvolume_subpath),) + if subvolume_subpath.strip() + if filesystem_mount_point.strip() ) diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 6e4fbf7d..f983a66d 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -18,12 +18,30 @@ def test_get_subvolumes_for_filesystem_parses_subvolume_list_output(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return( - 'ID 270 gen 107 top level 5 path subvol1\n' 'ID 272 gen 74 top level 5 path subvol2\n' + 'ID 270 gen 107 top level 5 path subvol1\nID 272 gen 74 top level 5 path subvol2\n' ) assert module.get_subvolumes_for_filesystem('btrfs', '/mnt') == ('/mnt/subvol1', '/mnt/subvol2') +def test_get_subvolumes_for_filesystem_skips_empty_subvolume_paths(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('\n \nID 272 gen 74 top level 5 path subvol2\n') + + assert module.get_subvolumes_for_filesystem('btrfs', '/mnt') == ('/mnt/subvol2',) + + +def test_get_subvolumes_for_filesystem_skips_empty_filesystem_mount_points(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'ID 270 gen 107 top level 5 path subvol1\nID 272 gen 74 top level 5 path subvol2\n' + ) + + assert module.get_subvolumes_for_filesystem('btrfs', ' ') == () + + def test_get_subvolumes_collects_subvolumes_matching_source_directories_from_all_filesystems(): flexmock(module).should_receive('get_filesystem_mount_points').and_return(('/mnt1', '/mnt2')) flexmock(module).should_receive('get_subvolumes_for_filesystem').with_args( @@ -243,6 +261,57 @@ def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_source assert config == {'btrfs': {}} +def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns(): + source_directories = ['/foo', '/mnt/subvol1'] + config = {'btrfs': {}, 'exclude_patterns': ['/bar']} + flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1', '/mnt/subvol2')) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return( + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ) + flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol2').and_return( + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2' + ) + flexmock(module).should_receive('snapshot_subvolume').with_args( + 'btrfs', '/mnt/subvol1', '/mnt/subvol1/.borgmatic-1234/mnt/subvol1' + ).once() + flexmock(module).should_receive('snapshot_subvolume').with_args( + 'btrfs', '/mnt/subvol2', '/mnt/subvol2/.borgmatic-1234/mnt/subvol2' + ).once() + flexmock(module).should_receive('make_snapshot_exclude_path').with_args( + '/mnt/subvol1' + ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234') + flexmock(module).should_receive('make_snapshot_exclude_path').with_args( + '/mnt/subvol2' + ).and_return('/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234') + + assert ( + module.dump_data_sources( + hook_config=config['btrfs'], + config=config, + log_prefix='test', + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + source_directories=source_directories, + dry_run=False, + ) + == [] + ) + + assert source_directories == [ + '/foo', + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1', + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2', + ] + assert config == { + 'btrfs': {}, + 'exclude_patterns': [ + '/bar', + '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234', + '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234', + ], + } + + def test_remove_data_source_dumps_deletes_snapshots(): config = {'btrfs': {}} flexmock(module).should_receive('get_subvolumes').and_return(('/mnt/subvol1', '/mnt/subvol2')) From 37efaeae88f289e2da06bee32cf4d77005acd3ef Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 30 Nov 2024 10:55:30 -0800 Subject: [PATCH 9/9] Warn if Btrfs is configured but there are no Btrfs subvolumes detected (#251). --- borgmatic/hooks/data_source/btrfs.py | 8 ++++++-- borgmatic/hooks/data_source/zfs.py | 3 +++ tests/unit/hooks/data_source/test_zfs.py | 23 +++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 7ab0f12f..3d812356 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -171,14 +171,18 @@ def dump_data_sources( If this is a dry run, then don't actually snapshot anything. ''' dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'{log_prefix}: Snapshotting Btrfs datasets{dry_run_label}') + logger.info(f'{log_prefix}: Snapshotting Btrfs subvolumes{dry_run_label}') # Based on the configured source directories, determine Btrfs subvolumes to backup. btrfs_command = hook_config.get('btrfs_command', 'btrfs') findmnt_command = hook_config.get('findmnt_command', 'findmnt') + subvolumes = get_subvolumes(btrfs_command, findmnt_command, source_directories) + + if not subvolumes: + logger.warning(f'{log_prefix}: No Btrfs subvolumes found to snapshot{dry_run_label}') # Snapshot each subvolume, rewriting source directories to use their snapshot paths. - for subvolume_path in get_subvolumes(btrfs_command, findmnt_command, source_directories): + for subvolume_path in subvolumes: logger.debug(f'{log_prefix}: Creating Btrfs snapshot for {subvolume_path} subvolume') snapshot_path = make_snapshot_path(subvolume_path) diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index a9e44c9d..5f011a1c 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -148,6 +148,9 @@ def dump_data_sources( # Snapshot each dataset, rewriting source directories to use the snapshot paths. snapshot_name = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' + if not requested_datasets: + logger.warning(f'{log_prefix}: No ZFS datasets found to snapshot{dry_run_label}') + for dataset_name, mount_point in requested_datasets: full_snapshot_name = f'{dataset_name}@{snapshot_name}' logger.debug(f'{log_prefix}: Creating ZFS snapshot {full_snapshot_name}{dry_run_label}') diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index cbb8cc9d..2eea24f7 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -97,6 +97,29 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_source_directories() assert source_directories == [snapshot_mount_path] +def test_dump_data_sources_snapshots_with_no_datasets_skips_snapshots(): + flexmock(module).should_receive('get_datasets_to_backup').and_return(()) + flexmock(module.os).should_receive('getpid').and_return(1234) + flexmock(module).should_receive('snapshot_dataset').never() + flexmock(module).should_receive('mount_snapshot').never() + source_directories = ['/mnt/dataset'] + + assert ( + module.dump_data_sources( + hook_config={}, + config={'source_directories': '/mnt/dataset', 'zfs': {}}, + log_prefix='test', + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + source_directories=source_directories, + dry_run=False, + ) + == [] + ) + + assert source_directories == ['/mnt/dataset'] + + def test_dump_data_sources_uses_custom_commands(): flexmock(module).should_receive('get_datasets_to_backup').and_return( (('dataset', '/mnt/dataset'),)