Pass through several "borg list" flags (#193).

This commit is contained in:
Dan Helfman 2019-06-25 10:18:30 -07:00
parent 1676a98c51
commit c644270599
9 changed files with 236 additions and 28 deletions

4
NEWS
View File

@ -1,3 +1,7 @@
1.3.11.dev0
* #193: Pass through several "borg list" flags like --short, --format, --sort-by, --first, --last,
etc. via borgmatic list command-line flags.
1.3.10 1.3.10
* #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero. * #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero.

31
borgmatic/borg/flags.py Normal file
View File

@ -0,0 +1,31 @@
import itertools
def make_flags(name, value):
'''
Given a flag name and its value, return it formatted as Borg-compatible flags.
'''
if not value:
return ()
flag = '--{}'.format(name.replace('_', '-'))
if value is True:
return (flag,)
return (flag, str(value))
def make_flags_from_arguments(arguments, excludes=()):
'''
Given borgmatic command-line arguments as an instance of argparse.Namespace, and optionally a
list of named arguments to exclude, generate and return the corresponding Borg command-line
flags as a tuple.
'''
return tuple(
itertools.chain.from_iterable(
make_flags(name, value=getattr(arguments, name))
for name in vars(arguments)
if name not in excludes and not name.startswith('_')
)
)

View File

@ -1,27 +1,42 @@
import logging import logging
from borgmatic.borg.flags import make_flags, make_flags_from_arguments
from borgmatic.execute import execute_command from borgmatic.execute import execute_command
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def list_archives( def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
):
''' '''
Given a local or remote repository path and a storage config dict, display the output of listing Given a local or remote repository path, a storage config dict, and the arguments to the list
Borg archives in the repository or return JSON output. Or, if an archive name is given, listing action, display the output of listing Borg archives in the repository or return JSON output. Or,
the files in that archive. if an archive name is given, listing the files in that archive.
''' '''
lock_wait = storage_config.get('lock_wait', None) lock_wait = storage_config.get('lock_wait', None)
full_command = ( full_command = (
(local_path, 'list', '::'.join((repository, archive)) if archive else repository) (
+ (('--remote-path', remote_path) if remote_path else ()) local_path,
+ (('--lock-wait', str(lock_wait)) if lock_wait else ()) 'list',
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ()) '::'.join((repository, list_arguments.archive))
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ()) if list_arguments.archive
+ (('--json',) if json else ()) else repository,
)
+ (
('--info',)
if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json
else ()
)
+ (
('--debug', '--show-rc')
if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json
else ()
)
+ make_flags('remote-path', remote_path)
+ make_flags('lock-wait', lock_wait)
+ make_flags_from_arguments(list_arguments, excludes=('repository', 'archive'))
) )
return execute_command(full_command, output_log_level=None if json else logging.WARNING) return execute_command(
full_command, output_log_level=None if list_arguments.json else logging.WARNING
)

View File

@ -248,7 +248,7 @@ def parse_arguments(*unparsed_arguments):
'list', 'list',
aliases=SUBPARSER_ALIASES['list'], aliases=SUBPARSER_ALIASES['list'],
help='List archives', help='List archives',
description='List archives', description='List archives or the contents of an archive',
add_help=False, add_help=False,
) )
list_group = list_parser.add_argument_group('list arguments') list_group = list_parser.add_argument_group('list arguments')
@ -258,7 +258,38 @@ def parse_arguments(*unparsed_arguments):
) )
list_group.add_argument('--archive', help='Name of archive to operate on') list_group.add_argument('--archive', help='Name of archive to operate on')
list_group.add_argument( list_group.add_argument(
'--json', dest='json', default=False, action='store_true', help='Output results as JSON' '--short', default=False, action='store_true', help='Output only archive or path names'
)
list_group.add_argument('--format', help='Format for file listing')
list_group.add_argument(
'--json', default=False, action='store_true', help='Output results as JSON'
)
list_group.add_argument(
'-P', '--prefix', help='Only list archive names starting with this prefix'
)
list_group.add_argument(
'-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob'
)
list_group.add_argument(
'--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
)
list_group.add_argument(
'--first', metavar='N', help='List first N archives after other filters are applied'
)
list_group.add_argument(
'--last', metavar='N', help='List first N archives after other filters are applied'
)
list_group.add_argument(
'-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
)
list_group.add_argument(
'--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
)
list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
list_group.add_argument(
'--pattern-from',
metavar='FILENAME',
help='Include or exclude paths matching patterns from pattern file, one per line',
) )
list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')

View File

@ -171,10 +171,9 @@ def run_actions(
json_output = borg_list.list_archives( json_output = borg_list.list_archives(
repository, repository,
storage, storage,
arguments['list'].archive, list_arguments=arguments['list'],
local_path=local_path, local_path=local_path,
remote_path=remote_path, remote_path=remote_path,
json=arguments['list'].json,
) )
if json_output: if json_output:
yield json.loads(json_output) yield json.loads(json_output)

View File

@ -48,6 +48,17 @@ for sub_command in prune create check list info; do
| grep -v '^--stats$' \ | grep -v '^--stats$' \
| grep -v '^--verbose$' \ | grep -v '^--verbose$' \
| grep -v '^--warning$' \ | grep -v '^--warning$' \
| grep -v '^--exclude' \
| grep -v '^--exclude-from' \
| grep -v '^--first' \
| grep -v '^--format' \
| grep -v '^--glob-archives' \
| grep -v '^--last' \
| grep -v '^--list-format' \
| grep -v '^--patterns-from' \
| grep -v '^--prefix' \
| grep -v '^--short' \
| grep -v '^--sort-by' \
| grep -v '^-h$' \ | grep -v '^-h$' \
>> all_borg_flags >> all_borg_flags
done done

View File

@ -1,6 +1,6 @@
from setuptools import find_packages, setup from setuptools import find_packages, setup
VERSION = '1.3.10' VERSION = '1.3.11.dev0'
setup( setup(

View File

@ -0,0 +1,47 @@
from flexmock import flexmock
from borgmatic.borg import flags as module
def test_make_flags_formats_string_value():
assert module.make_flags('foo', 'bar') == ('--foo', 'bar')
def test_make_flags_formats_integer_value():
assert module.make_flags('foo', 3) == ('--foo', '3')
def test_make_flags_formats_true_value():
assert module.make_flags('foo', True) == ('--foo',)
def test_make_flags_omits_false_value():
assert module.make_flags('foo', False) == ()
def test_make_flags_formats_name_with_underscore():
assert module.make_flags('posix_me_harder', 'okay') == ('--posix-me-harder', 'okay')
def test_make_flags_from_arguments_flattens_multiple_arguments():
flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
flexmock(module).should_receive('make_flags').with_args('baz', 'quux').and_return(
('baz', 'quux')
)
arguments = flexmock(foo='bar', baz='quux')
assert module.make_flags_from_arguments(arguments) == ('foo', 'bar', 'baz', 'quux')
def test_make_flags_from_arguments_excludes_underscored_argument_names():
flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
arguments = flexmock(foo='bar', _baz='quux')
assert module.make_flags_from_arguments(arguments) == ('foo', 'bar')
def test_make_flags_from_arguments_omits_excludes():
flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
arguments = flexmock(foo='bar', baz='quux')
assert module.make_flags_from_arguments(arguments, excludes=('baz', 'other')) == ('foo', 'bar')

View File

@ -1,5 +1,6 @@
import logging import logging
import pytest
from flexmock import flexmock from flexmock import flexmock
from borgmatic.borg import list as module from borgmatic.borg import list as module
@ -14,7 +15,9 @@ def test_list_archives_calls_borg_with_parameters():
LIST_COMMAND, output_log_level=logging.WARNING LIST_COMMAND, output_log_level=logging.WARNING
) )
module.list_archives(repository='repo', storage_config={}) module.list_archives(
repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
)
def test_list_archives_with_log_info_calls_borg_with_info_parameter(): def test_list_archives_with_log_info_calls_borg_with_info_parameter():
@ -23,7 +26,9 @@ def test_list_archives_with_log_info_calls_borg_with_info_parameter():
) )
insert_logging_mock(logging.INFO) insert_logging_mock(logging.INFO)
module.list_archives(repository='repo', storage_config={}) module.list_archives(
repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
)
def test_list_archives_with_log_info_and_json_suppresses_most_borg_output(): def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
@ -32,7 +37,9 @@ def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
) )
insert_logging_mock(logging.INFO) insert_logging_mock(logging.INFO)
module.list_archives(repository='repo', storage_config={}, json=True) module.list_archives(
repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
)
def test_list_archives_with_log_debug_calls_borg_with_debug_parameter(): def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
@ -41,7 +48,9 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
) )
insert_logging_mock(logging.DEBUG) insert_logging_mock(logging.DEBUG)
module.list_archives(repository='repo', storage_config={}) module.list_archives(
repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
)
def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output(): def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
@ -50,7 +59,9 @@ def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
) )
insert_logging_mock(logging.DEBUG) insert_logging_mock(logging.DEBUG)
module.list_archives(repository='repo', storage_config={}, json=True) module.list_archives(
repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
)
def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
@ -59,7 +70,11 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
LIST_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING LIST_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING
) )
module.list_archives(repository='repo', storage_config=storage_config) module.list_archives(
repository='repo',
storage_config=storage_config,
list_arguments=flexmock(archive=None, json=False),
)
def test_list_archives_with_archive_calls_borg_with_archive_parameter(): def test_list_archives_with_archive_calls_borg_with_archive_parameter():
@ -68,7 +83,11 @@ def test_list_archives_with_archive_calls_borg_with_archive_parameter():
('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING
) )
module.list_archives(repository='repo', storage_config=storage_config, archive='archive') module.list_archives(
repository='repo',
storage_config=storage_config,
list_arguments=flexmock(archive='archive', json=False),
)
def test_list_archives_with_local_path_calls_borg_via_local_path(): def test_list_archives_with_local_path_calls_borg_via_local_path():
@ -76,7 +95,12 @@ def test_list_archives_with_local_path_calls_borg_via_local_path():
('borg1',) + LIST_COMMAND[1:], output_log_level=logging.WARNING ('borg1',) + LIST_COMMAND[1:], output_log_level=logging.WARNING
) )
module.list_archives(repository='repo', storage_config={}, local_path='borg1') module.list_archives(
repository='repo',
storage_config={},
list_arguments=flexmock(archive=None, json=False),
local_path='borg1',
)
def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters(): def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters():
@ -84,7 +108,51 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
LIST_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING LIST_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING
) )
module.list_archives(repository='repo', storage_config={}, remote_path='borg1') module.list_archives(
repository='repo',
storage_config={},
list_arguments=flexmock(archive=None, json=False),
remote_path='borg1',
)
def test_list_archives_with_short_calls_borg_with_short_parameter():
flexmock(module).should_receive('execute_command').with_args(
LIST_COMMAND + ('--short',), output_log_level=logging.WARNING
).and_return('[]')
module.list_archives(
repository='repo',
storage_config={},
list_arguments=flexmock(archive=None, json=False, short=True),
)
@pytest.mark.parametrize(
'argument_name',
(
'prefix',
'glob_archives',
'sort_by',
'first',
'last',
'exclude',
'exclude_from',
'pattern',
'pattern_from',
),
)
def test_list_archives_passes_through_arguments_to_borg(argument_name):
flexmock(module).should_receive('execute_command').with_args(
LIST_COMMAND + ('--' + argument_name.replace('_', '-'), 'value'),
output_log_level=logging.WARNING,
).and_return('[]')
module.list_archives(
repository='repo',
storage_config={},
list_arguments=flexmock(archive=None, json=False, **{argument_name: 'value'}),
)
def test_list_archives_with_json_calls_borg_with_json_parameter(): def test_list_archives_with_json_calls_borg_with_json_parameter():
@ -92,6 +160,8 @@ def test_list_archives_with_json_calls_borg_with_json_parameter():
LIST_COMMAND + ('--json',), output_log_level=None LIST_COMMAND + ('--json',), output_log_level=None
).and_return('[]') ).and_return('[]')
json_output = module.list_archives(repository='repo', storage_config={}, json=True) json_output = module.list_archives(
repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
)
assert json_output == '[]' assert json_output == '[]'