From e80e0a253cae0f2c291eca9bff17f8f18b20eaad Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 20 Dec 2023 09:17:41 -0800 Subject: [PATCH] Add configured repository labels to the JSON output for all actions (#800). --- NEWS | 1 + borgmatic/actions/create.py | 5 ++-- borgmatic/actions/info.py | 8 ++--- borgmatic/actions/json.py | 16 ++++++++++ borgmatic/actions/list.py | 12 ++++---- borgmatic/actions/rinfo.py | 8 ++--- borgmatic/actions/rlist.py | 8 ++--- tests/unit/actions/test_create.py | 49 ++++++++++++++++++++++++++++--- tests/unit/actions/test_info.py | 31 ++++++++++++++++++- tests/unit/actions/test_json.py | 25 ++++++++++++++++ tests/unit/actions/test_list.py | 31 ++++++++++++++++++- tests/unit/actions/test_rinfo.py | 25 +++++++++++++++- tests/unit/actions/test_rlist.py | 23 ++++++++++++++- 13 files changed, 214 insertions(+), 28 deletions(-) create mode 100644 borgmatic/actions/json.py create mode 100644 tests/unit/actions/test_json.py diff --git a/NEWS b/NEWS index 7efa967a..426b3cf7 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.8.6.dev0 * #794: Fix a traceback when the "repositories" option contains both strings and key/value pairs. + * #800: Add configured repository labels to the JSON output for all actions. 1.8.5 * #701: Add a "skip_actions" option to skip running particular actions, handy for append-only or diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 8ca1e3b0..2c9bc646 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -3,6 +3,7 @@ import json import logging import os +import borgmatic.actions.json import borgmatic.borg.create import borgmatic.borg.state import borgmatic.config.validate @@ -107,8 +108,8 @@ def run_create( list_files=create_arguments.list_files, stream_processes=stream_processes, ) - if json_output: # pragma: nocover - yield json.loads(json_output) + if json_output: + yield borgmatic.actions.json.parse_json(json_output, repository.get('label')) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_data_source_dumps', diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index b09f3ece..d0ac4a9c 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -1,7 +1,7 @@ -import json import logging import borgmatic.actions.arguments +import borgmatic.actions.json import borgmatic.borg.info import borgmatic.borg.rlist import borgmatic.config.validate @@ -26,7 +26,7 @@ def run_info( if info_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, info_arguments.repository ): - if not info_arguments.json: # pragma: nocover + if not info_arguments.json: logger.answer( f'{repository.get("label", repository["path"])}: Displaying archive summary information' ) @@ -48,5 +48,5 @@ def run_info( local_path, remote_path, ) - if json_output: # pragma: nocover - yield json.loads(json_output) + if json_output: + yield borgmatic.actions.json.parse_json(json_output, repository.get('label')) diff --git a/borgmatic/actions/json.py b/borgmatic/actions/json.py new file mode 100644 index 00000000..cf849f2b --- /dev/null +++ b/borgmatic/actions/json.py @@ -0,0 +1,16 @@ +import json + + +def parse_json(borg_json_output, label): + ''' + Given a Borg JSON output string, parse it as JSON into a dict. Inject the given borgmatic + repository label into it and return the dict. + ''' + json_data = json.loads(borg_json_output) + + if 'repository' not in json_data: + return json_data + + json_data['repository']['label'] = label or '' + + return json_data diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index ae9da63c..96a62047 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -1,7 +1,7 @@ -import json import logging import borgmatic.actions.arguments +import borgmatic.actions.json import borgmatic.borg.list import borgmatic.config.validate @@ -25,10 +25,10 @@ def run_list( if list_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, list_arguments.repository ): - if not list_arguments.json: # pragma: nocover - if list_arguments.find_paths: + if not list_arguments.json: + if list_arguments.find_paths: # pragma: no cover logger.answer(f'{repository.get("label", repository["path"])}: Searching archives') - elif not list_arguments.archive: + elif not list_arguments.archive: # pragma: no cover logger.answer(f'{repository.get("label", repository["path"])}: Listing archives') archive_name = borgmatic.borg.rlist.resolve_archive_name( @@ -49,5 +49,5 @@ def run_list( local_path, remote_path, ) - if json_output: # pragma: nocover - yield json.loads(json_output) + if json_output: + yield borgmatic.actions.json.parse_json(json_output, repository.get('label')) diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 00de8922..9f04ee60 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -1,6 +1,6 @@ -import json import logging +import borgmatic.actions.json import borgmatic.borg.rinfo import borgmatic.config.validate @@ -24,7 +24,7 @@ def run_rinfo( if rinfo_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, rinfo_arguments.repository ): - if not rinfo_arguments.json: # pragma: nocover + if not rinfo_arguments.json: logger.answer( f'{repository.get("label", repository["path"])}: Displaying repository summary information' ) @@ -38,5 +38,5 @@ def run_rinfo( local_path=local_path, remote_path=remote_path, ) - if json_output: # pragma: nocover - yield json.loads(json_output) + if json_output: + yield borgmatic.actions.json.parse_json(json_output, repository.get('label')) diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index a79920b6..686838af 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -1,6 +1,6 @@ -import json import logging +import borgmatic.actions.json import borgmatic.borg.rlist import borgmatic.config.validate @@ -24,7 +24,7 @@ def run_rlist( if rlist_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, rlist_arguments.repository ): - if not rlist_arguments.json: # pragma: nocover + if not rlist_arguments.json: logger.answer(f'{repository.get("label", repository["path"])}: Listing repository') json_output = borgmatic.borg.rlist.list_repository( @@ -36,5 +36,5 @@ def run_rlist( local_path=local_path, remote_path=remote_path, ) - if json_output: # pragma: nocover - yield json.loads(json_output) + if json_output: + yield borgmatic.actions.json.parse_json(json_output, repository.get('label')) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 28685e2f..948c5190 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -19,7 +19,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): repository=None, progress=flexmock(), stats=flexmock(), - json=flexmock(), + json=False, list_files=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) @@ -54,7 +54,7 @@ def test_run_create_with_store_config_files_false_does_not_create_borgmatic_mani repository=None, progress=flexmock(), stats=flexmock(), - json=flexmock(), + json=False, list_files=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) @@ -91,7 +91,7 @@ def test_run_create_runs_with_selected_repository(): repository=flexmock(), progress=flexmock(), stats=flexmock(), - json=flexmock(), + json=False, list_files=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) @@ -123,7 +123,7 @@ def test_run_create_bails_if_repository_does_not_match(): repository=flexmock(), progress=flexmock(), stats=flexmock(), - json=flexmock(), + json=False, list_files=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) @@ -144,6 +144,47 @@ def test_run_create_bails_if_repository_does_not_match(): ) +def test_run_create_produces_json(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.borg.create).should_receive('create_archive').once().and_return( + flexmock() + ) + parsed_json = flexmock() + flexmock(module.borgmatic.actions.json).should_receive('parse_json').and_return(parsed_json) + flexmock(module).should_receive('create_borgmatic_manifest').once() + flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).and_return({}) + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + stats=flexmock(), + json=True, + list_files=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[]) + + assert list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={}, + hook_context={}, + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) == [parsed_json] + + def test_create_borgmatic_manifest_creates_manifest_file(): flexmock(module.os.path).should_receive('join').with_args( module.borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY, 'bootstrap', 'manifest.json' diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py index 1a5f5533..442eb999 100644 --- a/tests/unit/actions/test_info.py +++ b/tests/unit/actions/test_info.py @@ -13,7 +13,7 @@ def test_run_info_does_not_raise(): flexmock() ) flexmock(module.borgmatic.borg.info).should_receive('display_archives_info') - info_arguments = flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()) + info_arguments = flexmock(repository=flexmock(), archive=flexmock(), json=False) list( module.run_info( @@ -26,3 +26,32 @@ def test_run_info_does_not_raise(): remote_path=None, ) ) + + +def test_run_info_produces_json(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + flexmock() + ) + flexmock(module.borgmatic.actions.arguments).should_receive('update_arguments').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.info).should_receive('display_archives_info').and_return( + flexmock() + ) + parsed_json = flexmock() + flexmock(module.borgmatic.actions.json).should_receive('parse_json').and_return(parsed_json) + info_arguments = flexmock(repository=flexmock(), archive=flexmock(), json=True) + + assert list( + module.run_info( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + info_arguments=info_arguments, + global_arguments=flexmock(log_json=False), + local_path=None, + remote_path=None, + ) + ) == [parsed_json] diff --git a/tests/unit/actions/test_json.py b/tests/unit/actions/test_json.py new file mode 100644 index 00000000..9f7f839a --- /dev/null +++ b/tests/unit/actions/test_json.py @@ -0,0 +1,25 @@ +from flexmock import flexmock + +from borgmatic.actions import json as module + + +def test_parse_json_loads_json_from_string(): + flexmock(module.json).should_receive('loads').and_return({'repository': {'id': 'foo'}}) + + assert module.parse_json('{"repository": {"id": "foo"}}', label=None) == { + 'repository': {'id': 'foo', 'label': ''} + } + + +def test_parse_json_injects_label_into_parsed_data(): + flexmock(module.json).should_receive('loads').and_return({'repository': {'id': 'foo'}}) + + assert module.parse_json('{"repository": {"id": "foo"}}', label='bar') == { + 'repository': {'id': 'foo', 'label': 'bar'} + } + + +def test_parse_json_injects_nothing_when_repository_missing(): + flexmock(module.json).should_receive('loads').and_return({'stuff': {'id': 'foo'}}) + + assert module.parse_json('{"stuff": {"id": "foo"}}', label='bar') == {'stuff': {'id': 'foo'}} diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py index dd3b1326..c300d9f2 100644 --- a/tests/unit/actions/test_list.py +++ b/tests/unit/actions/test_list.py @@ -13,7 +13,9 @@ def test_run_list_does_not_raise(): flexmock() ) flexmock(module.borgmatic.borg.list).should_receive('list_archive') - list_arguments = flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()) + list_arguments = flexmock( + repository=flexmock(), archive=flexmock(), json=False, find_paths=None + ) list( module.run_list( @@ -26,3 +28,30 @@ def test_run_list_does_not_raise(): remote_path=None, ) ) + + +def test_run_list_produces_json(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return( + flexmock() + ) + flexmock(module.borgmatic.actions.arguments).should_receive('update_arguments').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.list).should_receive('list_archive').and_return(flexmock()) + parsed_json = flexmock() + flexmock(module.borgmatic.actions.json).should_receive('parse_json').and_return(parsed_json) + list_arguments = flexmock(repository=flexmock(), archive=flexmock(), json=True) + + assert list( + module.run_list( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), + local_path=None, + remote_path=None, + ) + ) == [parsed_json] diff --git a/tests/unit/actions/test_rinfo.py b/tests/unit/actions/test_rinfo.py index 4ba73c41..18e8cf86 100644 --- a/tests/unit/actions/test_rinfo.py +++ b/tests/unit/actions/test_rinfo.py @@ -7,7 +7,7 @@ def test_run_rinfo_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) flexmock(module.borgmatic.borg.rinfo).should_receive('display_repository_info') - rinfo_arguments = flexmock(repository=flexmock(), json=flexmock()) + rinfo_arguments = flexmock(repository=flexmock(), json=False) list( module.run_rinfo( @@ -20,3 +20,26 @@ def test_run_rinfo_does_not_raise(): remote_path=None, ) ) + + +def test_run_rinfo_parses_json(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.rinfo).should_receive('display_repository_info').and_return( + flexmock() + ) + parsed_json = flexmock() + flexmock(module.borgmatic.actions.json).should_receive('parse_json').and_return(parsed_json) + rinfo_arguments = flexmock(repository=flexmock(), json=True) + + list( + module.run_rinfo( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + rinfo_arguments=rinfo_arguments, + global_arguments=flexmock(log_json=False), + local_path=None, + remote_path=None, + ) + ) == [parsed_json] diff --git a/tests/unit/actions/test_rlist.py b/tests/unit/actions/test_rlist.py index 84798a76..34c1f75c 100644 --- a/tests/unit/actions/test_rlist.py +++ b/tests/unit/actions/test_rlist.py @@ -7,7 +7,7 @@ def test_run_rlist_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) flexmock(module.borgmatic.borg.rlist).should_receive('list_repository') - rlist_arguments = flexmock(repository=flexmock(), json=flexmock()) + rlist_arguments = flexmock(repository=flexmock(), json=False) list( module.run_rlist( @@ -20,3 +20,24 @@ def test_run_rlist_does_not_raise(): remote_path=None, ) ) + + +def test_run_rlist_produces_json(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.rlist).should_receive('list_repository').and_return(flexmock()) + parsed_json = flexmock() + flexmock(module.borgmatic.actions.json).should_receive('parse_json').and_return(parsed_json) + rlist_arguments = flexmock(repository=flexmock(), json=True) + + assert list( + module.run_rlist( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + rlist_arguments=rlist_arguments, + global_arguments=flexmock(), + local_path=None, + remote_path=None, + ) + ) == [parsed_json]