From c6442705993e7890c964b08cade1b9278dfd9d40 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 25 Jun 2019 10:18:30 -0700 Subject: [PATCH] Pass through several "borg list" flags (#193). --- NEWS | 4 ++ borgmatic/borg/flags.py | 31 +++++++++ borgmatic/borg/list.py | 41 ++++++++---- borgmatic/commands/arguments.py | 35 ++++++++++- borgmatic/commands/borgmatic.py | 3 +- scripts/find-unsupported-borg-options | 11 ++++ setup.py | 2 +- tests/unit/borg/test_flags.py | 47 ++++++++++++++ tests/unit/borg/test_list.py | 90 ++++++++++++++++++++++++--- 9 files changed, 236 insertions(+), 28 deletions(-) create mode 100644 borgmatic/borg/flags.py create mode 100644 tests/unit/borg/test_flags.py diff --git a/NEWS b/NEWS index c82aefc..3140e04 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.3.11.dev0 + * #193: Pass through several "borg list" flags like --short, --format, --sort-by, --first, --last, + etc. via borgmatic list command-line flags. + 1.3.10 * #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero. diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py new file mode 100644 index 0000000..4806f29 --- /dev/null +++ b/borgmatic/borg/flags.py @@ -0,0 +1,31 @@ +import itertools + + +def make_flags(name, value): + ''' + Given a flag name and its value, return it formatted as Borg-compatible flags. + ''' + if not value: + return () + + flag = '--{}'.format(name.replace('_', '-')) + + if value is True: + return (flag,) + + return (flag, str(value)) + + +def make_flags_from_arguments(arguments, excludes=()): + ''' + Given borgmatic command-line arguments as an instance of argparse.Namespace, and optionally a + list of named arguments to exclude, generate and return the corresponding Borg command-line + flags as a tuple. + ''' + return tuple( + itertools.chain.from_iterable( + make_flags(name, value=getattr(arguments, name)) + for name in vars(arguments) + if name not in excludes and not name.startswith('_') + ) + ) diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index c01f963..1dbb39d 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -1,27 +1,42 @@ import logging +from borgmatic.borg.flags import make_flags, make_flags_from_arguments from borgmatic.execute import execute_command logger = logging.getLogger(__name__) -def list_archives( - repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False -): +def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None): ''' - Given a local or remote repository path and a storage config dict, display the output of listing - Borg archives in the repository or return JSON output. Or, if an archive name is given, listing - the files in that archive. + Given a local or remote repository path, a storage config dict, and the arguments to the list + action, display the output of listing Borg archives in the repository or return JSON output. Or, + if an archive name is given, listing the files in that archive. ''' lock_wait = storage_config.get('lock_wait', None) full_command = ( - (local_path, 'list', '::'.join((repository, archive)) if archive else repository) - + (('--remote-path', remote_path) if remote_path else ()) - + (('--lock-wait', str(lock_wait)) if lock_wait else ()) - + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ()) - + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ()) - + (('--json',) if json else ()) + ( + local_path, + 'list', + '::'.join((repository, list_arguments.archive)) + if list_arguments.archive + else repository, + ) + + ( + ('--info',) + if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json + else () + ) + + ( + ('--debug', '--show-rc') + if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json + else () + ) + + make_flags('remote-path', remote_path) + + make_flags('lock-wait', lock_wait) + + make_flags_from_arguments(list_arguments, excludes=('repository', 'archive')) ) - return execute_command(full_command, output_log_level=None if json else logging.WARNING) + return execute_command( + full_command, output_log_level=None if list_arguments.json else logging.WARNING + ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 659e731..27c1e26 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -248,7 +248,7 @@ def parse_arguments(*unparsed_arguments): 'list', aliases=SUBPARSER_ALIASES['list'], help='List archives', - description='List archives', + description='List archives or the contents of an archive', add_help=False, ) list_group = list_parser.add_argument_group('list arguments') @@ -258,7 +258,38 @@ def parse_arguments(*unparsed_arguments): ) list_group.add_argument('--archive', help='Name of archive to operate on') list_group.add_argument( - '--json', dest='json', default=False, action='store_true', help='Output results as JSON' + '--short', default=False, action='store_true', help='Output only archive or path names' + ) + list_group.add_argument('--format', help='Format for file listing') + list_group.add_argument( + '--json', default=False, action='store_true', help='Output results as JSON' + ) + list_group.add_argument( + '-P', '--prefix', help='Only list archive names starting with this prefix' + ) + list_group.add_argument( + '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob' + ) + list_group.add_argument( + '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys' + ) + list_group.add_argument( + '--first', metavar='N', help='List first N archives after other filters are applied' + ) + list_group.add_argument( + '--last', metavar='N', help='List first N archives after other filters are applied' + ) + list_group.add_argument( + '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern' + ) + list_group.add_argument( + '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line' + ) + list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern') + list_group.add_argument( + '--pattern-from', + metavar='FILENAME', + help='Include or exclude paths matching patterns from pattern file, one per line', ) list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 95f8cb6..b282920 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -171,10 +171,9 @@ def run_actions( json_output = borg_list.list_archives( repository, storage, - arguments['list'].archive, + list_arguments=arguments['list'], local_path=local_path, remote_path=remote_path, - json=arguments['list'].json, ) if json_output: yield json.loads(json_output) diff --git a/scripts/find-unsupported-borg-options b/scripts/find-unsupported-borg-options index 3eeabf1..4958359 100755 --- a/scripts/find-unsupported-borg-options +++ b/scripts/find-unsupported-borg-options @@ -48,6 +48,17 @@ for sub_command in prune create check list info; do | grep -v '^--stats$' \ | grep -v '^--verbose$' \ | grep -v '^--warning$' \ + | grep -v '^--exclude' \ + | grep -v '^--exclude-from' \ + | grep -v '^--first' \ + | grep -v '^--format' \ + | grep -v '^--glob-archives' \ + | grep -v '^--last' \ + | grep -v '^--list-format' \ + | grep -v '^--patterns-from' \ + | grep -v '^--prefix' \ + | grep -v '^--short' \ + | grep -v '^--sort-by' \ | grep -v '^-h$' \ >> all_borg_flags done diff --git a/setup.py b/setup.py index acff927..eb45bcc 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.3.10' +VERSION = '1.3.11.dev0' setup( diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py new file mode 100644 index 0000000..c11b270 --- /dev/null +++ b/tests/unit/borg/test_flags.py @@ -0,0 +1,47 @@ +from flexmock import flexmock + +from borgmatic.borg import flags as module + + +def test_make_flags_formats_string_value(): + assert module.make_flags('foo', 'bar') == ('--foo', 'bar') + + +def test_make_flags_formats_integer_value(): + assert module.make_flags('foo', 3) == ('--foo', '3') + + +def test_make_flags_formats_true_value(): + assert module.make_flags('foo', True) == ('--foo',) + + +def test_make_flags_omits_false_value(): + assert module.make_flags('foo', False) == () + + +def test_make_flags_formats_name_with_underscore(): + assert module.make_flags('posix_me_harder', 'okay') == ('--posix-me-harder', 'okay') + + +def test_make_flags_from_arguments_flattens_multiple_arguments(): + flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar')) + flexmock(module).should_receive('make_flags').with_args('baz', 'quux').and_return( + ('baz', 'quux') + ) + arguments = flexmock(foo='bar', baz='quux') + + assert module.make_flags_from_arguments(arguments) == ('foo', 'bar', 'baz', 'quux') + + +def test_make_flags_from_arguments_excludes_underscored_argument_names(): + flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar')) + arguments = flexmock(foo='bar', _baz='quux') + + assert module.make_flags_from_arguments(arguments) == ('foo', 'bar') + + +def test_make_flags_from_arguments_omits_excludes(): + flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar')) + arguments = flexmock(foo='bar', baz='quux') + + assert module.make_flags_from_arguments(arguments, excludes=('baz', 'other')) == ('foo', 'bar') diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 2e6beb9..a68d7b7 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -1,5 +1,6 @@ import logging +import pytest from flexmock import flexmock from borgmatic.borg import list as module @@ -14,7 +15,9 @@ def test_list_archives_calls_borg_with_parameters(): LIST_COMMAND, output_log_level=logging.WARNING ) - module.list_archives(repository='repo', storage_config={}) + module.list_archives( + repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False) + ) def test_list_archives_with_log_info_calls_borg_with_info_parameter(): @@ -23,7 +26,9 @@ def test_list_archives_with_log_info_calls_borg_with_info_parameter(): ) insert_logging_mock(logging.INFO) - module.list_archives(repository='repo', storage_config={}) + module.list_archives( + repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False) + ) def test_list_archives_with_log_info_and_json_suppresses_most_borg_output(): @@ -32,7 +37,9 @@ def test_list_archives_with_log_info_and_json_suppresses_most_borg_output(): ) insert_logging_mock(logging.INFO) - module.list_archives(repository='repo', storage_config={}, json=True) + module.list_archives( + repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True) + ) def test_list_archives_with_log_debug_calls_borg_with_debug_parameter(): @@ -41,7 +48,9 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter(): ) insert_logging_mock(logging.DEBUG) - module.list_archives(repository='repo', storage_config={}) + module.list_archives( + repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False) + ) def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output(): @@ -50,7 +59,9 @@ def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output(): ) insert_logging_mock(logging.DEBUG) - module.list_archives(repository='repo', storage_config={}, json=True) + module.list_archives( + repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True) + ) def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): @@ -59,7 +70,11 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): LIST_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING ) - module.list_archives(repository='repo', storage_config=storage_config) + module.list_archives( + repository='repo', + storage_config=storage_config, + list_arguments=flexmock(archive=None, json=False), + ) def test_list_archives_with_archive_calls_borg_with_archive_parameter(): @@ -68,7 +83,11 @@ def test_list_archives_with_archive_calls_borg_with_archive_parameter(): ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING ) - module.list_archives(repository='repo', storage_config=storage_config, archive='archive') + module.list_archives( + repository='repo', + storage_config=storage_config, + list_arguments=flexmock(archive='archive', json=False), + ) def test_list_archives_with_local_path_calls_borg_via_local_path(): @@ -76,7 +95,12 @@ def test_list_archives_with_local_path_calls_borg_via_local_path(): ('borg1',) + LIST_COMMAND[1:], output_log_level=logging.WARNING ) - module.list_archives(repository='repo', storage_config={}, local_path='borg1') + module.list_archives( + repository='repo', + storage_config={}, + list_arguments=flexmock(archive=None, json=False), + local_path='borg1', + ) def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters(): @@ -84,7 +108,51 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters() LIST_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING ) - module.list_archives(repository='repo', storage_config={}, remote_path='borg1') + module.list_archives( + repository='repo', + storage_config={}, + list_arguments=flexmock(archive=None, json=False), + remote_path='borg1', + ) + + +def test_list_archives_with_short_calls_borg_with_short_parameter(): + flexmock(module).should_receive('execute_command').with_args( + LIST_COMMAND + ('--short',), output_log_level=logging.WARNING + ).and_return('[]') + + module.list_archives( + repository='repo', + storage_config={}, + list_arguments=flexmock(archive=None, json=False, short=True), + ) + + +@pytest.mark.parametrize( + 'argument_name', + ( + 'prefix', + 'glob_archives', + 'sort_by', + 'first', + 'last', + 'exclude', + 'exclude_from', + 'pattern', + 'pattern_from', + ), +) +def test_list_archives_passes_through_arguments_to_borg(argument_name): + flexmock(module).should_receive('execute_command').with_args( + LIST_COMMAND + ('--' + argument_name.replace('_', '-'), 'value'), + output_log_level=logging.WARNING, + ).and_return('[]') + + module.list_archives( + repository='repo', + storage_config={}, + list_arguments=flexmock(archive=None, json=False, **{argument_name: 'value'}), + ) def test_list_archives_with_json_calls_borg_with_json_parameter(): @@ -92,6 +160,8 @@ def test_list_archives_with_json_calls_borg_with_json_parameter(): LIST_COMMAND + ('--json',), output_log_level=None ).and_return('[]') - json_output = module.list_archives(repository='repo', storage_config={}, json=True) + json_output = module.list_archives( + repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True) + ) assert json_output == '[]'