From 789bcd402abe6bc052e1b3e64d4a7338567a83e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20L=C3=89VEIL?= Date: Sat, 28 Jul 2018 21:21:38 +0000 Subject: [PATCH] add support for `--list --json` (#74) --- borgmatic/borg/list.py | 7 +- borgmatic/commands/borgmatic.py | 136 +++++++++++------- .../integration/commands/test_borgmatic.py | 11 ++ borgmatic/tests/unit/borg/test_list.py | 13 +- borgmatic/tests/unit/commands/__init__.py | 0 .../tests/unit/commands/test_borgmatic.py | 38 +++++ 6 files changed, 148 insertions(+), 57 deletions(-) create mode 100644 borgmatic/tests/unit/commands/__init__.py create mode 100644 borgmatic/tests/unit/commands/test_borgmatic.py diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 989f88174..4bc24f7da 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -7,7 +7,7 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS logger = logging.getLogger(__name__) -def list_archives(verbosity, repository, storage_config, local_path='borg', remote_path=None): +def list_archives(verbosity, repository, storage_config, local_path='borg', remote_path=None, json=False): ''' Given a verbosity flag, a local or remote repository path, and a storage config dict, list Borg archives in the repository. @@ -18,6 +18,7 @@ def list_archives(verbosity, repository, storage_config, local_path='borg', remo (local_path, 'list', repository) + (('--remote-path', remote_path) if remote_path else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + + (('--json',) if json else ()) + { VERBOSITY_SOME: ('--info',), VERBOSITY_LOTS: ('--debug',), @@ -25,4 +26,6 @@ def list_archives(verbosity, repository, storage_config, local_path='borg', remo ) logger.debug(' '.join(full_command)) - subprocess.check_call(full_command) + + output = subprocess.check_output(full_command) + return output.decode() if output is not None else None diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index ee225088c..c8993ba16 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -1,5 +1,5 @@ - from argparse import ArgumentParser +import json import logging import os from subprocess import CalledProcessError @@ -76,6 +76,13 @@ def parse_arguments(*arguments): action='store_true', help='Display summary information on archives', ) + parser.add_argument( + '--json', + dest='json', + default=False, + action='store_true', + help='Output results from the --list option as json', + ) parser.add_argument( '-n', '--dry-run', dest='dry_run', @@ -90,6 +97,9 @@ def parse_arguments(*arguments): args = parser.parse_args(arguments) + if args.json and not args.list: + raise ValueError("The --json option can only be used with the --list option") + # If any of the action flags are explicitly requested, leave them as-is. Otherwise, assume # defaults: Mutate the given arguments to enable the default actions. if args.prune or args.create or args.check or args.list or args.info: @@ -121,59 +131,7 @@ def run_configuration(config_filename, args): # pragma: no cover if args.create: hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup') - for unexpanded_repository in location['repositories']: - repository = os.path.expanduser(unexpanded_repository) - dry_run_label = ' (dry run; not making any changes)' if args.dry_run else '' - if args.prune: - logger.info('{}: Pruning archives{}'.format(repository, dry_run_label)) - borg_prune.prune_archives( - args.verbosity, - args.dry_run, - repository, - storage, - retention, - local_path=local_path, - remote_path=remote_path, - ) - if args.create: - logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) - borg_create.create_archive( - args.verbosity, - args.dry_run, - repository, - location, - storage, - local_path=local_path, - remote_path=remote_path, - ) - if args.check: - logger.info('{}: Running consistency checks'.format(repository)) - borg_check.check_archives( - args.verbosity, - repository, - storage, - consistency, - local_path=local_path, - remote_path=remote_path, - ) - if args.list: - logger.info('{}: Listing archives'.format(repository)) - borg_list.list_archives( - args.verbosity, - repository, - storage, - local_path=local_path, - remote_path=remote_path, - ) - if args.info: - logger.info('{}: Displaying summary info for archives'.format(repository)) - borg_info.display_archives_info( - args.verbosity, - repository, - storage, - local_path=local_path, - remote_path=remote_path, - ) + _run_commands(args, consistency, local_path, location, remote_path, retention, storage) if args.create: hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup') @@ -182,6 +140,76 @@ def run_configuration(config_filename, args): # pragma: no cover raise +def _run_commands(args, consistency, local_path, location, remote_path, retention, storage): + json_results = [] + for unexpanded_repository in location['repositories']: + _run_commands_on_repository(args, consistency, json_results, local_path, location, remote_path, retention, + storage, unexpanded_repository) + if args.json: + sys.stdout.write(json.dumps(json_results)) + + +def _run_commands_on_repository(args, consistency, json_results, local_path, location, remote_path, retention, storage, + unexpanded_repository): # pragma: no cover + repository = os.path.expanduser(unexpanded_repository) + dry_run_label = ' (dry run; not making any changes)' if args.dry_run else '' + if args.prune: + logger.info('{}: Pruning archives{}'.format(repository, dry_run_label)) + borg_prune.prune_archives( + args.verbosity, + args.dry_run, + repository, + storage, + retention, + local_path=local_path, + remote_path=remote_path, + ) + if args.create: + logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) + borg_create.create_archive( + args.verbosity, + args.dry_run, + repository, + location, + storage, + local_path=local_path, + remote_path=remote_path, + ) + if args.check: + logger.info('{}: Running consistency checks'.format(repository)) + borg_check.check_archives( + args.verbosity, + repository, + storage, + consistency, + local_path=local_path, + remote_path=remote_path, + ) + if args.list: + logger.info('{}: Listing archives'.format(repository)) + output = borg_list.list_archives( + args.verbosity, + repository, + storage, + local_path=local_path, + remote_path=remote_path, + json=args.json, + ) + if args.json: + json_results.append(json.loads(output)) + else: + sys.stdout.write(output) + if args.info: + logger.info('{}: Displaying summary info for archives'.format(repository)) + borg_info.display_archives_info( + args.verbosity, + repository, + storage, + local_path=local_path, + remote_path=remote_path, + ) + + def main(): # pragma: no cover try: configure_signals() diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py index 6207cdb54..8cd1f76a2 100644 --- a/borgmatic/tests/integration/commands/test_borgmatic.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -15,6 +15,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): assert parser.config_paths == config_paths assert parser.excludes_filename == None assert parser.verbosity is None + assert parser.json is False def test_parse_arguments_with_path_arguments_overrides_defaults(): @@ -47,6 +48,11 @@ def test_parse_arguments_with_verbosity_flag_overrides_default(): assert parser.verbosity == 1 +def test_parse_arguments_with_json_flag_overrides_default(): + parser = module.parse_arguments('--list', '--json') + assert parser.json is True + + def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) @@ -82,3 +88,8 @@ def test_parse_arguments_with_invalid_arguments_exits(): with pytest.raises(SystemExit): module.parse_arguments('--posix-me-harder') + + +def test_parse_arguments_with_json_flag_but_no_list_flag_raises_value_error(): + with pytest.raises(ValueError): + module.parse_arguments('--json') diff --git a/borgmatic/tests/unit/borg/test_list.py b/borgmatic/tests/unit/borg/test_list.py index 1c6a69f73..48bd095ab 100644 --- a/borgmatic/tests/unit/borg/test_list.py +++ b/borgmatic/tests/unit/borg/test_list.py @@ -8,7 +8,7 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS def insert_subprocess_mock(check_call_command, **kwargs): subprocess = flexmock(module.subprocess) - subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() + subprocess.should_receive('check_output').with_args(check_call_command, **kwargs).once() LIST_COMMAND = ('borg', 'list', 'repo') @@ -44,6 +44,17 @@ def test_list_archives_with_verbosity_lots_calls_borg_with_debug_parameter(): ) +def test_list_archives_with_json_calls_borg_with_json_parameter(): + insert_subprocess_mock(LIST_COMMAND + ('--json',)) + + module.list_archives( + verbosity=None, + repository='repo', + storage_config={}, + json=True, + ) + + def test_list_archives_with_local_path_calls_borg_via_local_path(): insert_subprocess_mock(('borg1',) + LIST_COMMAND[1:]) diff --git a/borgmatic/tests/unit/commands/__init__.py b/borgmatic/tests/unit/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/borgmatic/tests/unit/commands/test_borgmatic.py b/borgmatic/tests/unit/commands/test_borgmatic.py new file mode 100644 index 000000000..cab0b0f34 --- /dev/null +++ b/borgmatic/tests/unit/commands/test_borgmatic.py @@ -0,0 +1,38 @@ +from borgmatic.commands import borgmatic +from flexmock import flexmock +import json +import pytest +import sys + + +def test__run_commands_handles_multiple_json_outputs_in_array(): + # THEN + (flexmock(borgmatic) + .should_receive("_run_commands_on_repository") + .times(3) + .replace_with(lambda args, consistency, json_results, local_path, location, remote_path, retention, storage, + unexpanded_repository: json_results.append({"whatever": unexpanded_repository})) + ) + + (flexmock(sys.stdout) + .should_call("write") + .with_args(json.dumps(json.loads(''' + [ + {"whatever": "fake_repo1"}, + {"whatever": "fake_repo2"}, + {"whatever": "fake_repo3"} + ] + '''))) + ) + + borgmatic._run_commands(args=flexmock(json=True), + consistency=None, + local_path=None, + location={"repositories": [ + "fake_repo1", + "fake_repo2", + "fake_repo3" + ]}, + remote_path=None, + retention=None, + storage=None)