From cc04bf57dfedb26bfa663d512bfd97dc92962a91 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 15 Aug 2022 15:04:40 -0700 Subject: [PATCH] Update list action for Borg 2 support, add rinfo action, and update extract consistency check for Borg 2. --- borgmatic/borg/check.py | 2 +- borgmatic/borg/extract.py | 40 +-- borgmatic/borg/list.py | 150 ++++---- borgmatic/borg/rlist.py | 121 +++++++ borgmatic/commands/arguments.py | 49 ++- borgmatic/commands/borgmatic.py | 90 ++++- docs/Dockerfile | 2 +- docs/how-to/backup-your-databases.md | 7 +- docs/how-to/extract-a-backup.md | 7 +- docs/how-to/inspect-your-backups.md | 12 +- docs/how-to/monitor-your-backups.md | 6 +- docs/how-to/run-arbitrary-borg-commands.md | 9 +- tests/unit/borg/test_extract.py | 143 ++++---- tests/unit/borg/test_list.py | 391 +++++++++++++-------- tests/unit/borg/test_rlist.py | 381 ++++++++++++++++++++ tests/unit/commands/test_borgmatic.py | 33 +- 16 files changed, 1082 insertions(+), 361 deletions(-) create mode 100644 borgmatic/borg/rlist.py create mode 100644 tests/unit/borg/test_rlist.py diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 917967413..e3dd0c32b 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -323,6 +323,6 @@ def check_archives( if 'extract' in checks: extract.extract_last_archive_dry_run( - storage_config, repository, lock_wait, local_path, remote_path + storage_config, local_borg_version, repository, lock_wait, local_path, remote_path ) write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract')) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index c86f556a5..5e2a58d2c 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -2,14 +2,19 @@ import logging import os import subprocess -from borgmatic.borg import environment, feature, flags +from borgmatic.borg import environment, feature, flags, rlist from borgmatic.execute import DO_NOT_CAPTURE, execute_command logger = logging.getLogger(__name__) def extract_last_archive_dry_run( - storage_config, repository, lock_wait=None, local_path='borg', remote_path=None + storage_config, + local_borg_version, + repository, + lock_wait=None, + local_path='borg', + remote_path=None, ): ''' Perform an extraction dry-run of the most recent archive. If there are no archives, skip the @@ -23,40 +28,23 @@ def extract_last_archive_dry_run( elif logger.isEnabledFor(logging.INFO): verbosity_flags = ('--info',) - full_list_command = ( - (local_path, 'list', '--short') - + remote_path_flags - + lock_wait_flags - + verbosity_flags - + (repository,) - ) - - borg_environment = environment.make_environment(storage_config) - - list_output = execute_command( - full_list_command, - output_log_level=None, - borg_local_path=local_path, - extra_environment=borg_environment, - ) - try: - last_archive_name = list_output.strip().splitlines()[-1] - except IndexError: + last_archive_name = rlist.resolve_archive_name( + repository, 'latest', storage_config, local_borg_version, local_path, remote_path + ) + except ValueError: + logger.warning('No archives found. Skipping extract consistency check.') return list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else () + borg_environment = environment.make_environment(storage_config) full_extract_command = ( (local_path, 'extract', '--dry-run') + remote_path_flags + lock_wait_flags + verbosity_flags + list_flag - + ( - '{repository}::{last_archive_name}'.format( - repository=repository, last_archive_name=last_archive_name - ), - ) + + flags.make_repository_archive_flags(repository, last_archive_name, local_borg_version) ) execute_command( diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 1d897ec73..fadc5055d 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -1,58 +1,24 @@ +import argparse import copy import logging import re -from borgmatic.borg import environment -from borgmatic.borg.flags import make_flags, make_flags_from_arguments +from borgmatic.borg import environment, feature, flags, rlist from borgmatic.execute import execute_command logger = logging.getLogger(__name__) -def resolve_archive_name(repository, archive, storage_config, local_path='borg', remote_path=None): - ''' - Given a local or remote repository path, an archive name, a storage config dict, a local Borg - path, and a remote Borg path, simply return the archive name. But if the archive name is - "latest", then instead introspect the repository for the latest archive and return its name. - - Raise ValueError if "latest" is given but there are no archives in the repository. - ''' - if archive != "latest": - return archive - - lock_wait = storage_config.get('lock_wait', None) - - full_command = ( - (local_path, 'list') - + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) - + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) - + make_flags('remote-path', remote_path) - + make_flags('lock-wait', lock_wait) - + make_flags('last', 1) - + ('--short', repository) - ) - - output = execute_command( - full_command, - output_log_level=None, - borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), - ) - try: - latest_archive = output.strip().splitlines()[-1] - except IndexError: - raise ValueError('No archives found in the repository') - - logger.debug('{}: Latest archive is {}'.format(repository, latest_archive)) - - return latest_archive - - MAKE_FLAGS_EXCLUDES = ('repository', 'archive', 'successful', 'paths', 'find_paths') def make_list_command( - repository, storage_config, list_arguments, local_path='borg', remote_path=None + repository, + storage_config, + local_borg_version, + list_arguments, + local_path='borg', + remote_path=None, ): ''' Given a local or remote repository path, a storage config dict, the arguments to the list @@ -73,13 +39,15 @@ def make_list_command( if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json else () ) - + make_flags('remote-path', remote_path) - + make_flags('lock-wait', lock_wait) - + make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES,) + + flags.make_flags('remote-path', remote_path) + + flags.make_flags('lock-wait', lock_wait) + + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES,) + ( - ('::'.join((repository, list_arguments.archive)),) + flags.make_repository_archive_flags( + repository, list_arguments.archive, local_borg_version + ) if list_arguments.archive - else (repository,) + else flags.make_repository_flags(repository, local_borg_version) ) + (tuple(list_arguments.paths) if list_arguments.paths else ()) ) @@ -109,29 +77,76 @@ def make_find_paths(find_paths): ) -def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None): +def list_archive( + repository, + storage_config, + local_borg_version, + list_arguments, + local_path='borg', + remote_path=None, +): ''' - Given a local or remote repository path, a storage config dict, the arguments to the list - action, and local and remote Borg paths, display the output of listing Borg archives in the - repository or return JSON output. Or, if an archive name is given, list the files in that - archive. Or, if list_arguments.find_paths are given, list the files by searching across multiple - archives. + Given a local or remote repository path, a storage config dict, the local Borg version, the + arguments to the list action, and local and remote Borg paths, display the output of listing + the files of a Borg archive (or return JSON output). If list_arguments.find_paths are given, + list the files by searching across multiple archives. If neither find_paths nor archive name + are given, instead list the archives in the given repository. ''' + if not list_arguments.archive and not list_arguments.find_paths: + if feature.available(feature.Feature.RLIST, local_borg_version): + logger.warning( + 'Omitting the --archive flag on the list action is deprecated when using Borg 2.x. Use the rlist action instead.' + ) + + rlist_arguments = argparse.Namespace( + repository=repository, + short=list_arguments.short, + format=list_arguments.format, + json=list_arguments.json, + prefix=list_arguments.prefix, + glob_archives=list_arguments.glob_archives, + sort_by=list_arguments.sort_by, + first=list_arguments.first, + last=list_arguments.last, + ) + return rlist.list_repository( + repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path + ) + + if feature.available(feature.Feature.RLIST, local_borg_version): + for flag_name in ('prefix', 'glob-archives', 'sort-by', 'first', 'last'): + if getattr(list_arguments, flag_name.replace('-', '_'), None): + raise ValueError( + f'The --{flag_name} flag on the list action is not supported when using the --archive flag and Borg 2.x.' + ) + borg_environment = environment.make_environment(storage_config) # If there are any paths to find (and there's not a single archive already selected), start by # getting a list of archives to search. if list_arguments.find_paths and not list_arguments.archive: - repository_arguments = copy.copy(list_arguments) - repository_arguments.archive = None - repository_arguments.json = False - repository_arguments.format = None + rlist_arguments = argparse.Namespace( + repository=repository, + short=True, + format=None, + json=None, + prefix=list_arguments.prefix, + glob_archives=list_arguments.glob_archives, + sort_by=list_arguments.sort_by, + first=list_arguments.first, + last=list_arguments.last, + ) # Ask Borg to list archives. Capture its output for use below. archive_lines = tuple( execute_command( - make_list_command( - repository, storage_config, repository_arguments, local_path, remote_path + rlist.make_rlist_command( + repository, + storage_config, + local_borg_version, + rlist_arguments, + local_path, + remote_path, ), output_log_level=None, borg_local_path=local_path, @@ -144,19 +159,18 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg', archive_lines = (list_arguments.archive,) # For each archive listed by Borg, run list on the contents of that archive. - for archive_line in archive_lines: - try: - archive = archive_line.split()[0] - except (AttributeError, IndexError): - archive = None - - if archive: - logger.warning(archive_line) + for archive in archive_lines: + logger.warning(f'{repository}: Listing archive {archive}') archive_arguments = copy.copy(list_arguments) archive_arguments.archive = archive main_command = make_list_command( - repository, storage_config, archive_arguments, local_path, remote_path + repository, + storage_config, + local_borg_version, + archive_arguments, + local_path, + remote_path, ) + make_find_paths(list_arguments.find_paths) output = execute_command( diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py new file mode 100644 index 000000000..a4cd1ef3e --- /dev/null +++ b/borgmatic/borg/rlist.py @@ -0,0 +1,121 @@ +import logging + +from borgmatic.borg import environment, feature, flags +from borgmatic.execute import execute_command + +logger = logging.getLogger(__name__) + + +def resolve_archive_name( + repository, archive, storage_config, local_borg_version, local_path='borg', remote_path=None +): + ''' + Given a local or remote repository path, an archive name, a storage config dict, a local Borg + path, and a remote Borg path, simply return the archive name. But if the archive name is + "latest", then instead introspect the repository for the latest archive and return its name. + + Raise ValueError if "latest" is given but there are no archives in the repository. + ''' + if archive != "latest": + return archive + + lock_wait = storage_config.get('lock_wait', None) + + full_command = ( + ( + local_path, + 'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list', + ) + + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + flags.make_flags('remote-path', remote_path) + + flags.make_flags('lock-wait', lock_wait) + + flags.make_flags('last', 1) + + ('--short',) + + flags.make_repository_flags(repository, local_borg_version) + ) + + output = execute_command( + full_command, + output_log_level=None, + borg_local_path=local_path, + extra_environment=environment.make_environment(storage_config), + ) + try: + latest_archive = output.strip().splitlines()[-1] + except IndexError: + raise ValueError('No archives found in the repository') + + logger.debug('{}: Latest archive is {}'.format(repository, latest_archive)) + + return latest_archive + + +MAKE_FLAGS_EXCLUDES = ('repository',) + + +def make_rlist_command( + repository, + storage_config, + local_borg_version, + rlist_arguments, + local_path='borg', + remote_path=None, +): + ''' + Given a local or remote repository path, a storage config dict, the local Borg version, the + arguments to the rlist action, and local and remote Borg paths, return a command as a tuple to + list archives with a repository. + ''' + lock_wait = storage_config.get('lock_wait', None) + + return ( + ( + local_path, + 'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list', + ) + + ( + ('--info',) + if logger.getEffectiveLevel() == logging.INFO and not rlist_arguments.json + else () + ) + + ( + ('--debug', '--show-rc') + if logger.isEnabledFor(logging.DEBUG) and not rlist_arguments.json + else () + ) + + flags.make_flags('remote-path', remote_path) + + flags.make_flags('lock-wait', lock_wait) + + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES,) + + flags.make_repository_flags(repository, local_borg_version) + ) + + +def list_repository( + repository, + storage_config, + local_borg_version, + rlist_arguments, + local_path='borg', + remote_path=None, +): + ''' + Given a local or remote repository path, a storage config dict, the local Borg version, the + arguments to the list action, and local and remote Borg paths, display the output of listing + Borg archives in the given repository (or return JSON output). + ''' + borg_environment = environment.make_environment(storage_config) + + main_command = make_rlist_command( + repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path + ) + + output = execute_command( + main_command, + output_log_level=None if rlist_arguments.json else logging.WARNING, + borg_local_path=local_path, + extra_environment=borg_environment, + ) + + if rlist_arguments.json: + return output diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index f4144531e..02180def7 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -14,6 +14,7 @@ SUBPARSER_ALIASES = { 'mount': ['--mount', '-m'], 'umount': ['--umount', '-u'], 'restore': ['--restore', '-r'], + 'rlist': [], 'list': ['--list', '-l'], 'rinfo': [], 'info': ['--info', '-i'], @@ -546,18 +547,54 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + rlist_parser = subparsers.add_parser( + 'rlist', + aliases=SUBPARSER_ALIASES['rlist'], + help='List repository', + description='List the archives in a repository', + add_help=False, + ) + rlist_group = rlist_parser.add_argument_group('rlist arguments') + rlist_group.add_argument( + '--repository', help='Path of repository to list, defaults to the configured repositories', + ) + rlist_group.add_argument( + '--short', default=False, action='store_true', help='Output only archive names' + ) + rlist_group.add_argument('--format', help='Format for archive listing') + rlist_group.add_argument( + '--json', default=False, action='store_true', help='Output results as JSON' + ) + rlist_group.add_argument( + '-P', '--prefix', help='Only list archive names starting with this prefix' + ) + rlist_group.add_argument( + '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob' + ) + rlist_group.add_argument( + '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys' + ) + rlist_group.add_argument( + '--first', metavar='N', help='List first N archives after other filters are applied' + ) + rlist_group.add_argument( + '--last', metavar='N', help='List last N archives after other filters are applied' + ) + rlist_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') + list_parser = subparsers.add_parser( 'list', aliases=SUBPARSER_ALIASES['list'], - help='List archives', - description='List archives or the contents of an archive', + help='List archive', + description='List the files in an archive or search for a file across archives', add_help=False, ) list_group = list_parser.add_argument_group('list arguments') list_group.add_argument( - '--repository', help='Path of repository to list, defaults to the configured repositories', + '--repository', + help='Path of repository containing archive to list, defaults to the configured repositories', ) - list_group.add_argument('--archive', help='Name of archive to list (or "latest")') + list_group.add_argument('--archive', help='Name of the archive to list (or "latest")') list_group.add_argument( '--path', metavar='PATH', @@ -573,7 +610,7 @@ def make_parsers(): help='Partial paths or patterns to search for and list across multiple archives', ) list_group.add_argument( - '--short', default=False, action='store_true', help='Output only archive or path names' + '--short', default=False, action='store_true', help='Output only path names' ) list_group.add_argument('--format', help='Format for file listing') list_group.add_argument( @@ -589,7 +626,7 @@ def make_parsers(): '--successful', default=True, action='store_true', - help='Deprecated in favor of listing successful (non-checkpoint) backups by default in newer versions of Borg', + help='Deprecated; no effect. Newer versions of Borg list successful (non-checkpoint) archives by default.', ) list_group.add_argument( '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys' diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index f806e4d35..e4455daa9 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -25,6 +25,7 @@ from borgmatic.borg import mount as borg_mount from borgmatic.borg import prune as borg_prune from borgmatic.borg import rcreate as borg_rcreate from borgmatic.borg import rinfo as borg_rinfo +from borgmatic.borg import rlist as borg_rlist from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments @@ -434,8 +435,13 @@ def run_actions( borg_extract.extract_archive( global_arguments.dry_run, repository, - borg_list.resolve_archive_name( - repository, arguments['extract'].archive, storage, local_path, remote_path + borg_rlist.resolve_archive_name( + repository, + arguments['extract'].archive, + storage, + local_borg_version, + local_path, + remote_path, ), arguments['extract'].paths, location, @@ -467,8 +473,13 @@ def run_actions( borg_export_tar.export_tar_archive( global_arguments.dry_run, repository, - borg_list.resolve_archive_name( - repository, arguments['export-tar'].archive, storage, local_path, remote_path + borg_rlist.resolve_archive_name( + repository, + arguments['export-tar'].archive, + storage, + local_borg_version, + local_path, + remote_path, ), arguments['export-tar'].paths, arguments['export-tar'].destination, @@ -492,8 +503,13 @@ def run_actions( borg_mount.mount_archive( repository, - borg_list.resolve_archive_name( - repository, arguments['mount'].archive, storage, local_path, remote_path + borg_rlist.resolve_archive_name( + repository, + arguments['mount'].archive, + storage, + local_borg_version, + local_path, + remote_path, ), arguments['mount'].mount_point, arguments['mount'].paths, @@ -525,8 +541,13 @@ def run_actions( if 'all' in restore_names: restore_names = [] - archive_name = borg_list.resolve_archive_name( - repository, arguments['restore'].archive, storage, local_path, remote_path + archive_name = borg_rlist.resolve_archive_name( + repository, + arguments['restore'].archive, + storage, + local_borg_version, + local_path, + remote_path, ) found_names = set() @@ -596,20 +617,45 @@ def run_actions( ', '.join(missing_names) ) ) - + if 'rlist' in arguments: + if arguments['rlist'].repository is None or validate.repositories_match( + repository, arguments['rlist'].repository + ): + rlist_arguments = copy.copy(arguments['rlist']) + if not rlist_arguments.json: # pragma: nocover + logger.warning('{}: Listing repository'.format(repository)) + json_output = borg_rlist.list_repository( + repository, + storage, + local_borg_version, + rlist_arguments=rlist_arguments, + local_path=local_path, + remote_path=remote_path, + ) + if json_output: # pragma: nocover + yield json.loads(json_output) if 'list' in arguments: if arguments['list'].repository is None or validate.repositories_match( repository, arguments['list'].repository ): list_arguments = copy.copy(arguments['list']) if not list_arguments.json: # pragma: nocover - logger.warning('{}: Listing archives'.format(repository)) - list_arguments.archive = borg_list.resolve_archive_name( - repository, list_arguments.archive, storage, local_path, remote_path + if list_arguments.find_paths: + logger.warning('{}: Searching archives'.format(repository)) + else: + logger.warning('{}: Listing archive'.format(repository)) + list_arguments.archive = borg_rlist.resolve_archive_name( + repository, + list_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, ) - json_output = borg_list.list_archives( + json_output = borg_list.list_archive( repository, storage, + local_borg_version, list_arguments=list_arguments, local_path=local_path, remote_path=remote_path, @@ -640,8 +686,13 @@ def run_actions( info_arguments = copy.copy(arguments['info']) if not info_arguments.json: # pragma: nocover logger.warning('{}: Displaying archive summary information'.format(repository)) - info_arguments.archive = borg_list.resolve_archive_name( - repository, info_arguments.archive, storage, local_path, remote_path + info_arguments.archive = borg_rlist.resolve_archive_name( + repository, + info_arguments.archive, + storage, + local_borg_version, + local_path, + remote_path, ) json_output = borg_info.display_archives_info( repository, @@ -658,8 +709,13 @@ def run_actions( repository, arguments['borg'].repository ): logger.warning('{}: Running arbitrary Borg command'.format(repository)) - archive_name = borg_list.resolve_archive_name( - repository, arguments['borg'].archive, storage, local_path, remote_path + archive_name = borg_rlist.resolve_archive_name( + repository, + arguments['borg'].archive, + storage, + local_borg_version, + local_path, + remote_path, ) borg_borg.run_arbitrary_borg( repository, diff --git a/docs/Dockerfile b/docs/Dockerfile index 89cb1862d..35d50b77e 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -4,7 +4,7 @@ COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ - && for action in init prune compact create check extract export-tar mount umount restore list info borg; do \ + && for action in rcreate prune compact create check extract export-tar mount umount restore rlist list rinfo info borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic "$action" --help >> /command-line.txt; done diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 01a84f963..6a485289a 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -133,14 +133,13 @@ that you'd like supported. To restore a database dump from an archive, use the `borgmatic restore` action. But the first step is to figure out which archive to restore from. A -good way to do that is to use the `list` action: +good way to do that is to use the `rlist` action: ```bash -borgmatic list +borgmatic rlist ``` -(No borgmatic `list` action? Try the old-style `--list`, or upgrade -borgmatic!) +(No borgmatic `rlist` action? Try `list` instead or upgrade borgmatic!) That should yield output looking something like: diff --git a/docs/how-to/extract-a-backup.md b/docs/how-to/extract-a-backup.md index 0dd47b0cc..a8ae9c544 100644 --- a/docs/how-to/extract-a-backup.md +++ b/docs/how-to/extract-a-backup.md @@ -9,14 +9,13 @@ eleventyNavigation: When the worst happens—or you want to test your backups—the first step is to figure out which archive to extract. A good way to do that is to use the -`list` action: +`rlist` action: ```bash -borgmatic list +borgmatic rlist ``` -(No borgmatic `list` action? Try the old-style `--list`, or upgrade -borgmatic!) +(No borgmatic `rlist` action? Try `list` instead or upgrade borgmatic!) That should yield output looking something like: diff --git a/docs/how-to/inspect-your-backups.md b/docs/how-to/inspect-your-backups.md index b266745f2..48ab194a9 100644 --- a/docs/how-to/inspect-your-backups.md +++ b/docs/how-to/inspect-your-backups.md @@ -46,14 +46,20 @@ borgmatic list borgmatic info ``` -New in borgmatic version 2.0.0 -There's also an `rinfo` action for displaying repository information with Borg -2.x: +New in borgmatic version 1.7.0 +There are also `rlist` and `rinfo` actions for displaying repository +information with Borg 2.x: ```bash +borgmatic rlist borgmatic rinfo ``` +See the [borgmatic command-line +reference](https://torsion.org/borgmatic/docs/reference/command-line/) for +more information. + + ### Searching for a file New in version 1.6.3 Let's say diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 3dabb7de2..f8c21bb26 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -329,9 +329,9 @@ output only shows up at the console, and not in syslog. ### Latest backups -All borgmatic actions that accept an "--archive" flag allow you to specify an -archive name of "latest". This lets you get the latest archive without having -to first run "borgmatic list" manually, which can be handy in automated +All borgmatic actions that accept an `--archive` flag allow you to specify an +archive name of `latest`. This lets you get the latest archive without having +to first run `borgmatic rlist` manually, which can be handy in automated scripts. Here's an example: ```bash diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index f0e152947..8ffddebf7 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -46,12 +46,11 @@ options, as that part is provided by borgmatic. You can also specify Borg options for relevant commands: ```bash -borgmatic borg list --progress +borgmatic borg rlist --short ``` -This runs Borg's `list` command once on each configured borgmatic -repository. However, the native `borgmatic list` action should be preferred -for most use. +This runs Borg's `rlist` command once on each configured borgmatic repository. +However, the native `borgmatic rlist` action should be preferred for most use. What if you only want to run Borg on a single configured borgmatic repository when you've got several configured? Not a problem. @@ -63,7 +62,7 @@ borgmatic borg --repository repo.borg break-lock And what about a single archive? ```bash -borgmatic borg --archive your-archive-name list +borgmatic borg --archive your-archive-name rlist ``` ### Limitations diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index 9c8cfa449..8e54dbc03 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -23,101 +23,100 @@ def insert_execute_command_output_mock(command, result): def test_extract_last_archive_dry_run_calls_borg_with_last_archive(): - insert_execute_command_output_mock( - ('borg', 'list', '--short', 'repo'), result='archive1\narchive2\n' - ) - insert_execute_command_mock(('borg', 'extract', '--dry-run', 'repo::archive2')) - flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') + insert_execute_command_mock(('borg', 'extract', '--dry-run', 'repo::archive')) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( - ('repo::archive2',) - ) - - module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=None) - - -def test_extract_last_archive_dry_run_without_any_archives_should_not_raise(): - insert_execute_command_output_mock(('borg', 'list', '--short', 'repo'), result='\n') - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(('repo',)) - - module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=None) - - -def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_parameter(): - insert_execute_command_output_mock( - ('borg', 'list', '--short', '--info', 'repo'), result='archive1\narchive2\n' - ) - insert_execute_command_mock(('borg', 'extract', '--dry-run', '--info', 'repo::archive2')) - insert_logging_mock(logging.INFO) - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( - ('repo::archive2',) - ) - - module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=None) - - -def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_parameter(): - insert_execute_command_output_mock( - ('borg', 'list', '--short', '--debug', '--show-rc', 'repo'), result='archive1\narchive2\n' - ) - insert_execute_command_mock( - ('borg', 'extract', '--dry-run', '--debug', '--show-rc', '--list', 'repo::archive2') - ) - insert_logging_mock(logging.DEBUG) - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( - ('repo::archive2',) - ) - - module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=None) - - -def test_extract_last_archive_dry_run_calls_borg_via_local_path(): - insert_execute_command_output_mock( - ('borg1', 'list', '--short', 'repo'), result='archive1\narchive2\n' - ) - insert_execute_command_mock(('borg1', 'extract', '--dry-run', 'repo::archive2')) - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( - ('repo::archive2',) + ('repo::archive',) ) module.extract_last_archive_dry_run( - storage_config={}, repository='repo', lock_wait=None, local_path='borg1' + storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + ) + + +def test_extract_last_archive_dry_run_without_any_archives_should_not_raise(): + flexmock(module.rlist).should_receive('resolve_archive_name').and_raise(ValueError) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(('repo',)) + + module.extract_last_archive_dry_run( + storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + ) + + +def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_parameter(): + flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') + insert_execute_command_mock(('borg', 'extract', '--dry-run', '--info', 'repo::archive')) + insert_logging_mock(logging.INFO) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + + module.extract_last_archive_dry_run( + storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + ) + + +def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_parameter(): + flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') + insert_execute_command_mock( + ('borg', 'extract', '--dry-run', '--debug', '--show-rc', '--list', 'repo::archive') + ) + insert_logging_mock(logging.DEBUG) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + + module.extract_last_archive_dry_run( + storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=None + ) + + +def test_extract_last_archive_dry_run_calls_borg_via_local_path(): + flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') + insert_execute_command_mock(('borg1', 'extract', '--dry-run', 'repo::archive')) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + + module.extract_last_archive_dry_run( + storage_config={}, + local_borg_version='1.2.3', + repository='repo', + lock_wait=None, + local_path='borg1', ) def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): - insert_execute_command_output_mock( - ('borg', 'list', '--short', '--remote-path', 'borg1', 'repo'), result='archive1\narchive2\n' - ) + flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') insert_execute_command_mock( - ('borg', 'extract', '--dry-run', '--remote-path', 'borg1', 'repo::archive2') + ('borg', 'extract', '--dry-run', '--remote-path', 'borg1', 'repo::archive') ) - flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( - ('repo::archive2',) + ('repo::archive',) ) module.extract_last_archive_dry_run( - storage_config={}, repository='repo', lock_wait=None, remote_path='borg1' + storage_config={}, + local_borg_version='1.2.3', + repository='repo', + lock_wait=None, + remote_path='borg1', ) def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): - insert_execute_command_output_mock( - ('borg', 'list', '--short', '--lock-wait', '5', 'repo'), result='archive1\narchive2\n' - ) + flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') insert_execute_command_mock( - ('borg', 'extract', '--dry-run', '--lock-wait', '5', 'repo::archive2') + ('borg', 'extract', '--dry-run', '--lock-wait', '5', 'repo::archive') ) - flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( - ('repo::archive2',) + ('repo::archive',) ) - module.extract_last_archive_dry_run(storage_config={}, repository='repo', lock_wait=5) + module.extract_last_archive_dry_run( + storage_config={}, local_borg_version='1.2.3', repository='repo', lock_wait=5 + ) def test_extract_archive_calls_borg_with_path_parameters(): diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 6123a077b..7111cb61a 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -8,129 +8,17 @@ from borgmatic.borg import list as module from ..test_verbosity import insert_logging_mock -BORG_LIST_LATEST_ARGUMENTS = ( - '--last', - '1', - '--short', - 'repo', -) - - -def test_resolve_archive_name_passes_through_non_latest_archive_name(): - archive = 'myhost-2030-01-01T14:41:17.647620' - - assert module.resolve_archive_name('repo', archive, storage_config={}) == archive - - -def test_resolve_archive_name_calls_borg_with_parameters(): - expected_archive = 'archive-name' - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - output_log_level=None, - borg_local_path='borg', - extra_environment=None, - ).and_return(expected_archive + '\n') - - assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive - - -def test_resolve_archive_name_with_log_info_calls_borg_with_info_parameter(): - expected_archive = 'archive-name' - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', '--info') + BORG_LIST_LATEST_ARGUMENTS, - output_log_level=None, - borg_local_path='borg', - extra_environment=None, - ).and_return(expected_archive + '\n') - insert_logging_mock(logging.INFO) - - assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive - - -def test_resolve_archive_name_with_log_debug_calls_borg_with_debug_parameter(): - expected_archive = 'archive-name' - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', '--debug', '--show-rc') + BORG_LIST_LATEST_ARGUMENTS, - output_log_level=None, - borg_local_path='borg', - extra_environment=None, - ).and_return(expected_archive + '\n') - insert_logging_mock(logging.DEBUG) - - assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive - - -def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): - expected_archive = 'archive-name' - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, - output_log_level=None, - borg_local_path='borg1', - extra_environment=None, - ).and_return(expected_archive + '\n') - - assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_path='borg1') - == expected_archive - ) - - -def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_parameters(): - expected_archive = 'archive-name' - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS, - output_log_level=None, - borg_local_path='borg', - extra_environment=None, - ).and_return(expected_archive + '\n') - - assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, remote_path='borg1') - == expected_archive - ) - - -def test_resolve_archive_name_without_archives_raises(): - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - output_log_level=None, - borg_local_path='borg', - extra_environment=None, - ).and_return('') - - with pytest.raises(ValueError): - module.resolve_archive_name('repo', 'latest', storage_config={}) - - -def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameters(): - expected_archive = 'archive-name' - - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS, - output_log_level=None, - borg_local_path='borg', - extra_environment=None, - ).and_return(expected_archive + '\n') - - assert ( - module.resolve_archive_name('repo', 'latest', storage_config={'lock_wait': 'okay'}) - == expected_archive - ) - def test_make_list_command_includes_log_info(): insert_logging_mock(logging.INFO) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), ) @@ -139,10 +27,14 @@ def test_make_list_command_includes_log_info(): def test_make_list_command_includes_json_but_not_info(): insert_logging_mock(logging.INFO) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), ) @@ -151,10 +43,14 @@ def test_make_list_command_includes_json_but_not_info(): def test_make_list_command_includes_log_debug(): insert_logging_mock(logging.DEBUG) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), ) @@ -163,10 +59,14 @@ def test_make_list_command_includes_log_debug(): def test_make_list_command_includes_json_but_not_debug(): insert_logging_mock(logging.DEBUG) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), ) @@ -174,9 +74,14 @@ def test_make_list_command_includes_json_but_not_debug(): def test_make_list_command_includes_json(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), ) @@ -184,9 +89,16 @@ def test_make_list_command_includes_json(): def test_make_list_command_includes_lock_wait(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( + ('--lock-wait', '5') + ) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + command = module.make_list_command( repository='repo', storage_config={'lock_wait': 5}, + local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), ) @@ -194,9 +106,16 @@ def test_make_list_command_includes_lock_wait(): def test_make_list_command_includes_archive(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=None, json=False), ) @@ -204,9 +123,16 @@ def test_make_list_command_includes_archive(): def test_make_list_command_includes_archive_and_path(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False), ) @@ -214,9 +140,14 @@ def test_make_list_command_includes_archive_and_path(): def test_make_list_command_includes_local_path(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), local_path='borg2', ) @@ -225,9 +156,16 @@ def test_make_list_command_includes_local_path(): def test_make_list_command_includes_remote_path(): + flexmock(module.flags).should_receive('make_flags').and_return( + ('--remote-path', 'borg2') + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), remote_path='borg2', ) @@ -236,9 +174,14 @@ def test_make_list_command_includes_remote_path(): def test_make_list_command_includes_short(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--short',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False, short=True), ) @@ -260,16 +203,23 @@ def test_make_list_command_includes_short(): ), ) def test_make_list_command_includes_additional_flags(argument_name): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( + (f"--{argument_name.replace('_', '-')}", 'value') + ) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + command = module.make_list_command( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=flexmock( archive=None, paths=None, json=False, find_paths=None, format=None, - **{argument_name: 'value'} + **{argument_name: 'value'}, ), ) @@ -303,89 +253,109 @@ def test_make_find_paths_adds_globs_to_path_fragments(): assert module.make_find_paths(('foo.txt',)) == ('sh:**/*foo.txt*/**',) -def test_list_archives_calls_borg_with_parameters(): - list_arguments = argparse.Namespace(archive=None, paths=None, json=False, find_paths=None) +def test_list_archive_calls_borg_with_parameters(): + list_arguments = argparse.Namespace(archive='archive', paths=None, json=False, find_paths=None) + flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=list_arguments, local_path='borg', remote_path=None, - ).and_return(('borg', 'list', 'repo')) + ).and_return(('borg', 'list', 'repo::archive')) flexmock(module).should_receive('make_find_paths').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', 'repo'), + ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg', extra_environment=None, ).once() - module.list_archives( - repository='repo', storage_config={}, list_arguments=list_arguments, + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, ) -def test_list_archives_with_json_suppresses_most_borg_output(): - list_arguments = argparse.Namespace(archive=None, paths=None, json=True, find_paths=None) +def test_list_archive_with_json_suppresses_most_borg_output(): + list_arguments = argparse.Namespace(archive='archive', paths=None, json=True, find_paths=None) + flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=list_arguments, local_path='borg', remote_path=None, - ).and_return(('borg', 'list', 'repo')) + ).and_return(('borg', 'list', 'repo::archive')) flexmock(module).should_receive('make_find_paths').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', 'repo'), + ('borg', 'list', 'repo::archive'), output_log_level=None, borg_local_path='borg', extra_environment=None, ).once() - module.list_archives( - repository='repo', storage_config={}, list_arguments=list_arguments, + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, ) -def test_list_archives_calls_borg_with_local_path(): - list_arguments = argparse.Namespace(archive=None, paths=None, json=False, find_paths=None) +def test_list_archive_calls_borg_with_local_path(): + list_arguments = argparse.Namespace(archive='archive', paths=None, json=False, find_paths=None) + flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=list_arguments, local_path='borg2', remote_path=None, - ).and_return(('borg2', 'list', 'repo')) + ).and_return(('borg2', 'list', 'repo::archive')) flexmock(module).should_receive('make_find_paths').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg2', 'list', 'repo'), + ('borg2', 'list', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg2', extra_environment=None, ).once() - module.list_archives( - repository='repo', storage_config={}, list_arguments=list_arguments, local_path='borg2', + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, + local_path='borg2', ) -def test_list_archives_calls_borg_multiple_times_with_find_paths(): +def test_list_archive_calls_borg_multiple_times_with_find_paths(): glob_paths = ('**/*foo.txt*/**',) list_arguments = argparse.Namespace( - archive=None, paths=None, json=False, find_paths=['foo.txt'], format=None + archive=None, + json=False, + find_paths=['foo.txt'], + prefix=None, + glob_archives=None, + sort_by=None, + first=None, + last=None, ) - flexmock(module).should_receive('make_list_command').and_return( - ('borg', 'list', 'repo') - ).and_return(('borg', 'list', 'repo::archive1')).and_return(('borg', 'list', 'repo::archive2')) - flexmock(module).should_receive('make_find_paths').and_return(glob_paths) - flexmock(module.environment).should_receive('make_environment') + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.rlist).should_receive('make_rlist_command').and_return(('borg', 'list', 'repo')) flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo'), output_log_level=None, @@ -394,6 +364,10 @@ def test_list_archives_calls_borg_multiple_times_with_find_paths(): ).and_return( 'archive1 Sun, 2022-05-29 15:27:04 [abc]\narchive2 Mon, 2022-05-30 19:47:15 [xyz]' ).once() + flexmock(module).should_receive('make_list_command').and_return( + ('borg', 'list', 'repo::archive1') + ).and_return(('borg', 'list', 'repo::archive2')) + flexmock(module).should_receive('make_find_paths').and_return(glob_paths) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive1') + glob_paths, @@ -408,17 +382,22 @@ def test_list_archives_calls_borg_multiple_times_with_find_paths(): extra_environment=None, ).once() - module.list_archives( - repository='repo', storage_config={}, list_arguments=list_arguments, + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, ) -def test_list_archives_calls_borg_with_archive(): +def test_list_archive_calls_borg_with_archive(): list_arguments = argparse.Namespace(archive='archive', paths=None, json=False, find_paths=None) + flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( repository='repo', storage_config={}, + local_borg_version='1.2.3', list_arguments=list_arguments, local_path='borg', remote_path=None, @@ -432,6 +411,124 @@ def test_list_archives_calls_borg_with_archive(): extra_environment=None, ).once() - module.list_archives( - repository='repo', storage_config={}, list_arguments=list_arguments, + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, + ) + + +def test_list_archive_without_archive_delegates_to_list_repository(): + list_arguments = argparse.Namespace( + archive=None, + short=None, + format=None, + json=None, + prefix=None, + glob_archives=None, + sort_by=None, + first=None, + last=None, + find_paths=None, + ) + + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.rlist).should_receive('list_repository') + flexmock(module.environment).should_receive('make_environment').never() + flexmock(module).should_receive('execute_command').never() + + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, + ) + + +def test_list_archive_with_borg_features_without_archive_delegates_to_list_repository(): + list_arguments = argparse.Namespace( + archive=None, + short=None, + format=None, + json=None, + prefix=None, + glob_archives=None, + sort_by=None, + first=None, + last=None, + find_paths=None, + ) + + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.rlist).should_receive('list_repository') + flexmock(module.environment).should_receive('make_environment').never() + flexmock(module).should_receive('execute_command').never() + + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, + ) + + +@pytest.mark.parametrize( + 'archive_filter_flag', ('prefix', 'glob_archives', 'sort_by', 'first', 'last',), +) +def test_list_archive_with_archive_disallows_archive_filter_flag_if_rlist_feature_available( + archive_filter_flag, +): + list_arguments = argparse.Namespace( + archive='archive', paths=None, json=False, find_paths=None, **{archive_filter_flag: 'foo'} + ) + + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.RLIST, '1.2.3' + ).and_return(True) + + with pytest.raises(ValueError): + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, + ) + + +@pytest.mark.parametrize( + 'archive_filter_flag', ('prefix', 'glob_archives', 'sort_by', 'first', 'last',), +) +def test_list_archive_with_archive_allows_archive_filter_flag_if_rlist_feature_unavailable( + archive_filter_flag, +): + list_arguments = argparse.Namespace( + archive='archive', paths=None, json=False, find_paths=None, **{archive_filter_flag: 'foo'} + ) + + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.RLIST, '1.2.3' + ).and_return(False) + flexmock(module).should_receive('make_list_command').with_args( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, + local_path='borg', + remote_path=None, + ).and_return(('borg', 'list', 'repo::archive')) + flexmock(module).should_receive('make_find_paths').and_return(()) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'list', 'repo::archive'), + output_log_level=logging.WARNING, + borg_local_path='borg', + extra_environment=None, + ).once() + + module.list_archive( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=list_arguments, ) diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py new file mode 100644 index 000000000..816843551 --- /dev/null +++ b/tests/unit/borg/test_rlist.py @@ -0,0 +1,381 @@ +import argparse +import logging + +import pytest +from flexmock import flexmock + +from borgmatic.borg import rlist as module + +from ..test_verbosity import insert_logging_mock + +BORG_LIST_LATEST_ARGUMENTS = ( + '--last', + '1', + '--short', + 'repo', +) + + +def test_resolve_archive_name_passes_through_non_latest_archive_name(): + archive = 'myhost-2030-01-01T14:41:17.647620' + + assert ( + module.resolve_archive_name('repo', archive, storage_config={}, local_borg_version='1.2.3') + == archive + ) + + +def test_resolve_archive_name_calls_borg_with_parameters(): + expected_archive = 'archive-name' + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + output_log_level=None, + borg_local_path='borg', + extra_environment=None, + ).and_return(expected_archive + '\n') + + assert ( + module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + == expected_archive + ) + + +def test_resolve_archive_name_with_log_info_calls_borg_with_info_parameter(): + expected_archive = 'archive-name' + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'list', '--info') + BORG_LIST_LATEST_ARGUMENTS, + output_log_level=None, + borg_local_path='borg', + extra_environment=None, + ).and_return(expected_archive + '\n') + insert_logging_mock(logging.INFO) + + assert ( + module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + == expected_archive + ) + + +def test_resolve_archive_name_with_log_debug_calls_borg_with_debug_parameter(): + expected_archive = 'archive-name' + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'list', '--debug', '--show-rc') + BORG_LIST_LATEST_ARGUMENTS, + output_log_level=None, + borg_local_path='borg', + extra_environment=None, + ).and_return(expected_archive + '\n') + insert_logging_mock(logging.DEBUG) + + assert ( + module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + == expected_archive + ) + + +def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): + expected_archive = 'archive-name' + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, + output_log_level=None, + borg_local_path='borg1', + extra_environment=None, + ).and_return(expected_archive + '\n') + + assert ( + module.resolve_archive_name( + 'repo', 'latest', storage_config={}, local_borg_version='1.2.3', local_path='borg1' + ) + == expected_archive + ) + + +def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_parameters(): + expected_archive = 'archive-name' + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS, + output_log_level=None, + borg_local_path='borg', + extra_environment=None, + ).and_return(expected_archive + '\n') + + assert ( + module.resolve_archive_name( + 'repo', 'latest', storage_config={}, local_borg_version='1.2.3', remote_path='borg1' + ) + == expected_archive + ) + + +def test_resolve_archive_name_without_archives_raises(): + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, + output_log_level=None, + borg_local_path='borg', + extra_environment=None, + ).and_return('') + + with pytest.raises(ValueError): + module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + + +def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameters(): + expected_archive = 'archive-name' + + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS, + output_log_level=None, + borg_local_path='borg', + extra_environment=None, + ).and_return(expected_archive + '\n') + + assert ( + module.resolve_archive_name( + 'repo', 'latest', storage_config={'lock_wait': 'okay'}, local_borg_version='1.2.3' + ) + == expected_archive + ) + + +def test_make_rlist_command_includes_log_info(): + insert_logging_mock(logging.INFO) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False), + ) + + assert command == ('borg', 'list', '--info', 'repo') + + +def test_make_rlist_command_includes_json_but_not_info(): + insert_logging_mock(logging.INFO) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=True), + ) + + assert command == ('borg', 'list', '--json', 'repo') + + +def test_make_rlist_command_includes_log_debug(): + insert_logging_mock(logging.DEBUG) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False), + ) + + assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') + + +def test_make_rlist_command_includes_json_but_not_debug(): + insert_logging_mock(logging.DEBUG) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=True), + ) + + assert command == ('borg', 'list', '--json', 'repo') + + +def test_make_rlist_command_includes_json(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=True), + ) + + assert command == ('borg', 'list', '--json', 'repo') + + +def test_make_rlist_command_includes_lock_wait(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( + ('--lock-wait', '5') + ) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={'lock_wait': 5}, + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False), + ) + + assert command == ('borg', 'list', '--lock-wait', '5', 'repo') + + +def test_make_rlist_command_includes_local_path(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False), + local_path='borg2', + ) + + assert command == ('borg2', 'list', 'repo') + + +def test_make_rlist_command_includes_remote_path(): + flexmock(module.flags).should_receive('make_flags').and_return( + ('--remote-path', 'borg2') + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False), + remote_path='borg2', + ) + + assert command == ('borg', 'list', '--remote-path', 'borg2', 'repo') + + +def test_make_rlist_command_includes_short(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--short',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock(archive=None, paths=None, json=False, short=True), + ) + + assert command == ('borg', 'list', '--short', 'repo') + + +@pytest.mark.parametrize( + 'argument_name', + ( + 'prefix', + 'glob_archives', + 'sort_by', + 'first', + 'last', + 'exclude', + 'exclude_from', + 'pattern', + 'patterns_from', + ), +) +def test_make_rlist_command_includes_additional_flags(argument_name): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( + (f"--{argument_name.replace('_', '-')}", 'value') + ) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock( + archive=None, + paths=None, + json=False, + find_paths=None, + format=None, + **{argument_name: 'value'}, + ), + ) + + assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') + + +def test_list_repository_calls_borg_with_parameters(): + rlist_arguments = argparse.Namespace(json=False) + + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module).should_receive('make_rlist_command').with_args( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=rlist_arguments, + local_path='borg', + remote_path=None, + ).and_return(('borg', 'rlist', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'rlist', 'repo'), + output_log_level=logging.WARNING, + borg_local_path='borg', + extra_environment=None, + ).once() + + module.list_repository( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=rlist_arguments, + ) + + +def test_list_repository_with_json_returns_borg_output(): + rlist_arguments = argparse.Namespace(json=True) + json_output = flexmock() + + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module).should_receive('make_rlist_command').with_args( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=rlist_arguments, + local_path='borg', + remote_path=None, + ).and_return(('borg', 'rlist', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').and_return(json_output) + + assert ( + module.list_repository( + repository='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=rlist_arguments, + ) + == json_output + ) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index e17c20ef9..365400c50 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -571,10 +571,35 @@ def test_run_actions_does_not_raise_for_mount_action(): ) +def test_run_actions_does_not_raise_for_rlist_action(): + flexmock(module.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borg_rlist).should_receive('list_repository') + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False), + 'rlist': flexmock(repository=flexmock(), json=flexmock()), + } + + list( + module.run_actions( + arguments=arguments, + config_filename='test.yaml', + location={'repositories': ['repo']}, + storage={}, + retention={}, + consistency={}, + hooks={}, + local_path=None, + remote_path=None, + local_borg_version=None, + repository_path='repo', + ) + ) + + def test_run_actions_does_not_raise_for_list_action(): flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock()) - flexmock(module.borg_list).should_receive('list_archives') + flexmock(module.borg_rlist).should_receive('resolve_archive_name').and_return(flexmock()) + flexmock(module.borg_list).should_receive('list_archive') arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False), 'list': flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()), @@ -624,7 +649,7 @@ def test_run_actions_does_not_raise_for_rinfo_action(): def test_run_actions_does_not_raise_for_info_action(): flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock()) + flexmock(module.borg_rlist).should_receive('resolve_archive_name').and_return(flexmock()) flexmock(module.borg_info).should_receive('display_archives_info') arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False), @@ -650,7 +675,7 @@ def test_run_actions_does_not_raise_for_info_action(): def test_run_actions_does_not_raise_for_borg_action(): flexmock(module.validate).should_receive('repositories_match').and_return(True) - flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock()) + flexmock(module.borg_rlist).should_receive('resolve_archive_name').and_return(flexmock()) flexmock(module.borg_borg).should_receive('run_arbitrary_borg') arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False),