From d17b2c74dbf8dba351b9664955901ea33c8c34b5 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 18 Mar 2023 04:35:55 +0530 Subject: [PATCH 1/3] feat: add optional check for existence of source directories --- borgmatic/borg/create.py | 18 ++++++++++++++++++ borgmatic/config/schema.yaml | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 0889c3d1..f3c3c69b 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -306,6 +306,22 @@ def collect_special_file_paths( ) +def check_all_source_directories_exist(source_directories): + ''' + Given a sequence of source directories, check that they all exist. If any do not, raise an + exception. + ''' + missing_directories = [ + source_directory + for source_directory in source_directories + if not os.path.exists(source_directory) + ] + if missing_directories: + raise ValueError( + 'Source directories do not exist: {}'.format(', '.join(missing_directories)) + ) + + def create_archive( dry_run, repository, @@ -331,6 +347,8 @@ def create_archive( borgmatic_source_directories = expand_directories( collect_borgmatic_source_directories(location_config.get('borgmatic_source_directory')) ) + if location_config.get('source_directories_must_exist', False): + check_all_source_directories_exist(location_config.get('source_directories')) sources = deduplicate_directories( map_directories_to_devices( expand_directories( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 2f873b7a..0135299c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -202,6 +202,12 @@ properties: path prevents "borgmatic restore" from finding any database dumps created before the change. Defaults to ~/.borgmatic example: /tmp/borgmatic + source_directories_must_exist: + type: boolean + description: | + If true, then source directories must exist, otherwise an + error is raised. Defaults to false. + example: true storage: type: object description: | From 997f60b3e660a4b219c611d83d628c9bcf55626a Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 18 Mar 2023 17:24:21 +0530 Subject: [PATCH 2/3] add tests --- borgmatic/borg/create.py | 8 +++--- tests/unit/borg/test_create.py | 45 ++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index f3c3c69b..f459e51f 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -317,9 +317,7 @@ def check_all_source_directories_exist(source_directories): if not os.path.exists(source_directory) ] if missing_directories: - raise ValueError( - 'Source directories do not exist: {}'.format(', '.join(missing_directories)) - ) + raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}") def create_archive( @@ -509,7 +507,9 @@ def create_archive( ) elif output_log_level is None: return execute_command_and_capture_output( - create_command, working_directory=working_directory, extra_environment=borg_environment, + create_command, + working_directory=working_directory, + extra_environment=borg_environment, ) else: execute_command( diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 3c4e4338..835ebed1 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -207,7 +207,6 @@ def test_make_exclude_flags_includes_exclude_patterns_filename_when_given(): def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config(): - exclude_flags = module.make_exclude_flags( location_config={'exclude_from': ['excludes', 'other']} ) @@ -1054,7 +1053,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), + 'feature_available,option_flag', + ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), ) def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_parameters( feature_available, option_flag @@ -1189,7 +1189,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par @pytest.mark.parametrize( - 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner')), + 'feature_available,option_flag', + ((True, '--numeric-ids'), (False, '--numeric-owner')), ) def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( feature_available, option_flag @@ -1291,7 +1292,12 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter @pytest.mark.parametrize( 'option_name,option_value', - (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),), + ( + ('ctime', True), + ('ctime', False), + ('birthtime', True), + ('birthtime', False), + ), ) def test_create_archive_with_basic_option_calls_borg_with_corresponding_parameter( option_name, option_value @@ -1767,7 +1773,12 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--info', '--progress',), + ('borg', 'create') + + REPO_ARCHIVE_WITH_PATHS + + ( + '--info', + '--progress', + ), output_log_level=logging.INFO, output_file=module.DO_NOT_CAPTURE, borg_local_path='borg', @@ -2530,3 +2541,27 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read local_borg_version='1.2.3', stream_processes=processes, ) + + +def test_create_archive_with_non_existent_directory_and_source_directories_must_exist_raises_error(): + """ + If a source directory doesn't exist and source_directories_must_exist is True, raise an error. + """ + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module.os.path).should_receive('exists').and_return(False) + + with pytest.raises(ValueError): + module.create_archive( + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + 'source_directories_must_exist': True, + }, + storage_config={}, + local_borg_version='1.2.3', + ) From f803836416f15c50da237b1e419ee4fff1810654 Mon Sep 17 00:00:00 2001 From: Divyansh Singh Date: Sat, 18 Mar 2023 17:27:33 +0530 Subject: [PATCH 3/3] reformat --- borgmatic/borg/create.py | 4 +--- tests/unit/borg/test_create.py | 20 ++++---------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index f459e51f..87a0fdd7 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -507,9 +507,7 @@ def create_archive( ) elif output_log_level is None: return execute_command_and_capture_output( - create_command, - working_directory=working_directory, - extra_environment=borg_environment, + create_command, working_directory=working_directory, extra_environment=borg_environment, ) else: execute_command( diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 835ebed1..3a5bddc1 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -1053,8 +1053,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( @pytest.mark.parametrize( - 'feature_available,option_flag', - ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), + 'feature_available,option_flag', ((True, '--upload-ratelimit'), (False, '--remote-ratelimit')), ) def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_parameters( feature_available, option_flag @@ -1189,8 +1188,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par @pytest.mark.parametrize( - 'feature_available,option_flag', - ((True, '--numeric-ids'), (False, '--numeric-owner')), + 'feature_available,option_flag', ((True, '--numeric-ids'), (False, '--numeric-owner')), ) def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( feature_available, option_flag @@ -1292,12 +1290,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter @pytest.mark.parametrize( 'option_name,option_value', - ( - ('ctime', True), - ('ctime', False), - ('birthtime', True), - ('birthtime', False), - ), + (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),), ) def test_create_archive_with_basic_option_calls_borg_with_corresponding_parameter( option_name, option_value @@ -1773,12 +1766,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'create') - + REPO_ARCHIVE_WITH_PATHS - + ( - '--info', - '--progress', - ), + ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--info', '--progress',), output_log_level=logging.INFO, output_file=module.DO_NOT_CAPTURE, borg_local_path='borg',