Initial work on a Btrfs hook (#251).
This commit is contained in:
parent
86e5085acc
commit
b5b5c1fafa
3
NEWS
3
NEWS
@ -1,4 +1,7 @@
|
||||
1.9.4.dev0
|
||||
* #251 (beta): Add a Btrfs hook for snapshotting and backing up Btrfs subvolumes. See the
|
||||
documentation for more information:
|
||||
https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
|
||||
* #926: Fix library error when running within a PyInstaller bundle.
|
||||
* Reorganize data source and monitoring hooks to make developing new hooks easier.
|
||||
|
||||
|
||||
@ -33,10 +33,11 @@ def get_borgmatic_source_directory(config):
|
||||
TEMPORARY_DIRECTORY_PREFIX = 'borgmatic-'
|
||||
|
||||
|
||||
def replace_temporary_subdirectory_with_glob(path):
|
||||
def replace_temporary_subdirectory_with_glob(path, temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX):
|
||||
'''
|
||||
Given an absolute temporary directory path, look for a subdirectory within it starting with the
|
||||
temporary directory prefix and replace it with an appropriate glob. For instance, given:
|
||||
Given an absolute temporary directory path and an optional temporary directory prefix, look for
|
||||
a subdirectory within it starting with the temporary directory prefix (or a default) and replace
|
||||
it with an appropriate glob. For instance, given:
|
||||
|
||||
/tmp/borgmatic-aet8kn93/borgmatic
|
||||
|
||||
@ -50,8 +51,8 @@ def replace_temporary_subdirectory_with_glob(path):
|
||||
'/',
|
||||
*(
|
||||
(
|
||||
f'{TEMPORARY_DIRECTORY_PREFIX}*'
|
||||
if subdirectory.startswith(TEMPORARY_DIRECTORY_PREFIX)
|
||||
f'{temporary_directory_prefix}*'
|
||||
if subdirectory.startswith(temporary_directory_prefix)
|
||||
else subdirectory
|
||||
)
|
||||
for subdirectory in path.split(os.path.sep)
|
||||
|
||||
@ -2288,3 +2288,17 @@ properties:
|
||||
example: /usr/local/bin/umount
|
||||
description: |
|
||||
Configuration for integration with the ZFS filesystem.
|
||||
btrfs:
|
||||
type: ["object", "null"]
|
||||
additionalProperties: false
|
||||
properties:
|
||||
btrfs_command:
|
||||
type: string
|
||||
description: |
|
||||
Command to use instead of "btrfs".
|
||||
example: /usr/local/bin/btrfs
|
||||
findmnt_command:
|
||||
type: string
|
||||
description: |
|
||||
Command to use instead of "findmnt".
|
||||
example: /usr/local/bin/findmnt
|
||||
|
||||
283
borgmatic/hooks/data_source/btrfs.py
Normal file
283
borgmatic/hooks/data_source/btrfs.py
Normal file
@ -0,0 +1,283 @@
|
||||
import functools
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.execute
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def use_streaming(hook_config, config, log_prefix): # pragma: no cover
|
||||
'''
|
||||
Return whether dump streaming is used for this hook. (Spoiler: It isn't.)
|
||||
'''
|
||||
return False
|
||||
|
||||
|
||||
def get_subvolumes(btrfs_command, findmnt_command, source_directories=None):
|
||||
'''
|
||||
Given a Btrfs command to run and a sequence of configured source directories, find the
|
||||
intersection between the current Btrfs filesystem and subvolume mount points and the configured
|
||||
borgmatic source directories. The idea is that these are the requested subvolumes to snapshot.
|
||||
|
||||
If the source directories is None, then return all subvolumes.
|
||||
|
||||
Return the result as a sequence of matching subvolume mount points.
|
||||
'''
|
||||
# Find all top-level Btrfs filesystem mount points.
|
||||
findmnt_output = borgmatic.execute.execute_command_and_capture_output(
|
||||
(
|
||||
findmnt_command,
|
||||
'-nt',
|
||||
'btrfs',
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
filesystem_mount_points = tuple(
|
||||
line.rstrip().split(' ')[0]
|
||||
for line in findmnt_output.splitlines()
|
||||
)
|
||||
except ValueError:
|
||||
raise ValueError('Invalid {findmnt_command} output')
|
||||
|
||||
source_directories_set = set(source_directories or ())
|
||||
subvolumes = []
|
||||
|
||||
# For each filesystem mount point, find its subvolumes and match them again the given source
|
||||
# directories to find the subvolumes to backup. Also try to match the filesystem mount point
|
||||
# itself.
|
||||
for mount_point in filesystem_mount_points:
|
||||
if source_directories is None or mount_point in source_directories_set:
|
||||
subvolumes.append(mount_point)
|
||||
|
||||
btrfs_output = borgmatic.execute.execute_command_and_capture_output(
|
||||
(
|
||||
btrfs_command,
|
||||
'subvolume',
|
||||
'list',
|
||||
mount_point,
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
subvolumes.extend(
|
||||
subvolume_path
|
||||
for line in btrfs_output.splitlines()
|
||||
for subvolume_subpath in (line.rstrip().split(' ')[-1],)
|
||||
for subvolume_path in (os.path.join(mount_point, subvolume_subpath),)
|
||||
if source_directories is None or subvolume_path in source_directories_set
|
||||
)
|
||||
except ValueError:
|
||||
raise ValueError('Invalid {btrfs_command} subvolume list output')
|
||||
|
||||
return tuple(subvolumes)
|
||||
|
||||
|
||||
BORGMATIC_SNAPSHOT_PREFIX = '.borgmatic-snapshot-'
|
||||
|
||||
|
||||
def make_snapshot_path(subvolume_path): # pragma: no cover
|
||||
'''
|
||||
Given the path to a subvolume, make a corresponding snapshot path for it.
|
||||
'''
|
||||
return os.path.join(
|
||||
subvolume_path,
|
||||
f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
|
||||
'.', # Borg 1.4+ "slashdot" hack.
|
||||
# Included so that the snapshot ends up in the Borg archive at the "original" subvolume
|
||||
# path.
|
||||
subvolume_path.lstrip(os.path.sep),
|
||||
)
|
||||
|
||||
|
||||
def make_snapshot_exclude_path(subvolume_path): # pragma: no cover
|
||||
'''
|
||||
Given the path to a subvolume, make a corresponding exclude path for its embedded snapshot path.
|
||||
This is to work around a quirk of Btrfs: If you make a snapshot path as a child directory of a
|
||||
subvolume, then the snapshot's own initial directory component shows up as an empty directory
|
||||
within the snapshot itself. For instance, if you have a Btrfs subvolume at /mnt and make a
|
||||
snapshot of it at:
|
||||
|
||||
/mnt/.borgmatic-snapshot-1234/mnt
|
||||
|
||||
... then the snapshot itself will have an empty directory at:
|
||||
|
||||
/mnt/.borgmatic-snapshot-1234/mnt/.borgmatic-snapshot-1234
|
||||
|
||||
So to prevent that from ending up in the Borg archive, this function produces its path for
|
||||
exclusion.
|
||||
'''
|
||||
snapshot_directory = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}'
|
||||
|
||||
return os.path.join(
|
||||
subvolume_path,
|
||||
snapshot_directory,
|
||||
subvolume_path.lstrip(os.path.sep),
|
||||
snapshot_directory,
|
||||
)
|
||||
|
||||
|
||||
def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path): # pragma: no cover
|
||||
'''
|
||||
Given a Btrfs command to run, the path to a subvolume, and the path for a snapshot, create a new
|
||||
Btrfs snapshot of the subvolume.
|
||||
'''
|
||||
os.makedirs(os.path.dirname(snapshot_path), mode=0o700, exist_ok=True)
|
||||
|
||||
borgmatic.execute.execute_command(
|
||||
(
|
||||
btrfs_command,
|
||||
'subvolume',
|
||||
'snapshot',
|
||||
subvolume_path,
|
||||
snapshot_path,
|
||||
),
|
||||
output_log_level=logging.DEBUG,
|
||||
)
|
||||
|
||||
|
||||
def dump_data_sources(
|
||||
hook_config,
|
||||
config,
|
||||
log_prefix,
|
||||
config_paths,
|
||||
borgmatic_runtime_directory,
|
||||
source_directories,
|
||||
dry_run,
|
||||
):
|
||||
'''
|
||||
Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic
|
||||
configuration file paths, the borgmatic runtime directory, the configured source directories,
|
||||
and whether this is a dry run, auto-detect and snapshot any Btrfs subvolume mount points listed
|
||||
in the given source directories. Also update those source directories, replacing subvolume mount
|
||||
points with corresponding snapshot directories so they get stored in the Borg archive instead.
|
||||
Use the log prefix in any log entries.
|
||||
|
||||
Return an empty sequence, since there are no ongoing dump processes from this hook.
|
||||
|
||||
If this is a dry run, then don't actually snapshot anything.
|
||||
'''
|
||||
dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
|
||||
logger.info(f'{log_prefix}: Snapshotting Btrfs datasets{dry_run_label}')
|
||||
|
||||
# Based on the configured source directories, determine Btrfs subvolumes to backup.
|
||||
btrfs_command = hook_config.get('btrfs_command', 'btrfs')
|
||||
findmnt_command = hook_config.get('findmnt_command', 'findmnt')
|
||||
requested_subvolume_paths = get_subvolumes(btrfs_command, findmnt_command, source_directories)
|
||||
|
||||
# Snapshot each subvolume, rewriting source directories to use their snapshot paths.
|
||||
for subvolume_path in requested_subvolume_paths:
|
||||
logger.debug(f'{log_prefix}: Creating Btrfs snapshot for {subvolume_path} subvolume')
|
||||
|
||||
snapshot_path = make_snapshot_path(subvolume_path)
|
||||
|
||||
if dry_run:
|
||||
continue
|
||||
|
||||
snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path)
|
||||
|
||||
if subvolume_path in source_directories:
|
||||
source_directories.remove(subvolume_path)
|
||||
|
||||
source_directories.append(snapshot_path)
|
||||
config.setdefault('exclude_patterns', []).append(make_snapshot_exclude_path(subvolume_path))
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def delete_snapshot(btrfs_command, snapshot_path): # pragma: no cover
|
||||
'''
|
||||
Given a Btrfs command to run and the name of a snapshot path, delete it.
|
||||
'''
|
||||
borgmatic.execute.execute_command(
|
||||
(
|
||||
btrfs_command,
|
||||
'subvolume',
|
||||
'delete',
|
||||
snapshot_path,
|
||||
),
|
||||
output_log_level=logging.DEBUG,
|
||||
)
|
||||
|
||||
|
||||
def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_directory, dry_run):
|
||||
'''
|
||||
Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic runtime
|
||||
directory, and whether this is a dry run, delete any Btrfs snapshots created by borgmatic. Use
|
||||
the log prefix in any log entries. If this is a dry run, then don't actually remove anything.
|
||||
'''
|
||||
dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
|
||||
|
||||
btrfs_command = hook_config.get('btrfs_command', 'btrfs')
|
||||
findmnt_command = hook_config.get('findmnt_command', 'findmnt')
|
||||
|
||||
try:
|
||||
all_subvolume_paths = get_subvolumes(btrfs_command, findmnt_command)
|
||||
except FileNotFoundError as error:
|
||||
logger.debug(f'{log_prefix}: Could not find "{error.filename}" command')
|
||||
return
|
||||
except subprocess.CalledProcessError as error:
|
||||
logger.debug(f'{log_prefix}: {error}')
|
||||
return
|
||||
|
||||
for subvolume_path in all_subvolume_paths:
|
||||
subvolume_snapshots_glob = borgmatic.config.paths.replace_temporary_subdirectory_with_glob(
|
||||
os.path.normpath(make_snapshot_path(subvolume_path)),
|
||||
temporary_directory_prefix=BORGMATIC_SNAPSHOT_PREFIX,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f'{log_prefix}: Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}'
|
||||
)
|
||||
|
||||
for snapshot_path in glob.glob(subvolume_snapshots_glob):
|
||||
if not os.path.isdir(snapshot_path):
|
||||
continue
|
||||
|
||||
logger.debug(f'{log_prefix}: Deleting Btrfs snapshot {snapshot_path}{dry_run_label}')
|
||||
|
||||
if dry_run:
|
||||
continue
|
||||
|
||||
try:
|
||||
delete_snapshot(btrfs_command, snapshot_path)
|
||||
except FileNotFoundError:
|
||||
logger.debug(f'{log_prefix}: Could not find "{btrfs_command}" command')
|
||||
return
|
||||
except subprocess.CalledProcessError as error:
|
||||
logger.debug(f'{log_prefix}: {error}')
|
||||
return
|
||||
|
||||
# Strip off the subvolume path from the end of the snapshot path and then delete the
|
||||
# resulting directory.
|
||||
shutil.rmtree(snapshot_path.rsplit(subvolume_path, 1)[0])
|
||||
|
||||
|
||||
def make_data_source_dump_patterns(
|
||||
hook_config, config, log_prefix, borgmatic_runtime_directory, name=None
|
||||
): # pragma: no cover
|
||||
'''
|
||||
Restores aren't implemented, because stored files can be extracted directly with "extract".
|
||||
'''
|
||||
return ()
|
||||
|
||||
|
||||
def restore_data_source_dump(
|
||||
hook_config,
|
||||
config,
|
||||
log_prefix,
|
||||
data_source,
|
||||
dry_run,
|
||||
extract_process,
|
||||
connection_params,
|
||||
borgmatic_runtime_directory,
|
||||
): # pragma: no cover
|
||||
'''
|
||||
Restores aren't implemented, because stored files can be extracted directly with "extract".
|
||||
'''
|
||||
raise NotImplementedError()
|
||||
@ -104,6 +104,7 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path): # p
|
||||
first).
|
||||
'''
|
||||
os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True)
|
||||
|
||||
borgmatic.execute.execute_command(
|
||||
(
|
||||
mount_command,
|
||||
@ -131,8 +132,7 @@ def dump_data_sources(
|
||||
is a dry run, auto-detect and snapshot any ZFS dataset mount points listed in the given source
|
||||
directories and any dataset with a borgmatic-specific user property. Also update those source
|
||||
directories, replacing dataset mount points with corresponding snapshot directories so they get
|
||||
stored in the Borg archive instead of the dataset mount points. Use the log prefix in any log
|
||||
entries.
|
||||
stored in the Borg archive instead. Use the log prefix in any log entries.
|
||||
|
||||
Return an empty sequence, since there are no ongoing dump processes from this hook.
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user