From a31ce337e97850fbea8af19aaa1277c699b09bcf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Oct 2022 11:07:43 -0700 Subject: [PATCH] Skip auto-exclusion of special files when user explicitly sets read_special to true (#587). --- borgmatic/borg/create.py | 8 +- docs/how-to/backup-your-databases.md | 10 +- tests/unit/borg/test_create.py | 181 ++++++++++++++++++--------- 3 files changed, 138 insertions(+), 61 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 3fd3335bc..13f794264 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -332,7 +332,6 @@ def create_archive( upload_rate_limit = storage_config.get('upload_rate_limit', None) umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) - read_special = True if (location_config.get('read_special') or stream_processes) else False files_cache = location_config.get('files_cache') archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT) extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '') @@ -384,7 +383,7 @@ def create_archive( + atime_flags + (('--noctime',) if location_config.get('ctime') is False else ()) + (('--nobirthtime',) if location_config.get('birthtime') is False else ()) - + (('--read-special',) if read_special else ()) + + (('--read-special',) if location_config.get('read_special') or stream_processes else ()) + noflags_flags + (('--files-cache', files_cache) if files_cache else ()) + (('--remote-path', remote_path) if remote_path else ()) @@ -410,8 +409,9 @@ def create_archive( borg_environment = environment.make_environment(storage_config) - # If read_special is enabled, exclude files that might cause Borg to hang. - if read_special: + # If database hooks are enabled (as indicated by streaming processes), exclude files that might + # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True. + if stream_processes and not location_config.get('read_special'): logger.debug(f'{repository}: Collecting special file paths') special_file_paths = collect_special_file_paths( create_command, diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 77a7a0698..0bb3b8371 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -217,7 +217,11 @@ special files are excluded from backups (named pipes, block devices, character devices, and sockets) to prevent hanging. Try a command like `find /your/source/path -type b -or -type c -or -type p -or -type s` to find such files. Common directories to exclude are `/dev` and `/run`, but that may -not be exhaustive. +not be exhaustive. New in version +1.7.3 When database hooks are enabled, borgmatic automatically excludes +special files that may cause Borg to hang, so you no longer need to manually +exclude them. You can override/prevent this behavior by explicitly setting +`read_special` to true. ### Manual restoration @@ -273,3 +277,7 @@ Alternatively, if excluding special files is too onerous, you can create two separate borgmatic configuration files—one for your source files and a separate one for backing up databases. That way, the database `read_special` option will not be active when backing up special files. + +New in version 1.7.3 See +Limitations above about borgmatic's automatic exclusion of special files to +prevent Borg hangs. diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 076ee1890..c12011399 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -1147,62 +1147,6 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter ) -def test_create_archive_with_read_special_adds_special_files_to_excludes(): - flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) - flexmock(module).should_receive('map_directories_to_devices').and_return({}) - flexmock(module).should_receive('expand_directories').and_return(()) - flexmock(module).should_receive('pattern_root_directories').and_return([]) - flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) - flexmock(module).should_receive('expand_home_directories').and_return(()).and_return( - ('special',) - ) - flexmock(module).should_receive('write_pattern_file').and_return(None).and_return( - flexmock(name='/excludes') - ) - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module).should_receive('ensure_files_readable') - flexmock(module).should_receive('make_pattern_flags').and_return(()) - flexmock(module).should_receive('make_exclude_flags').and_return(()).and_return( - '--exclude-from', '/excludes' - ) - flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( - (f'repo::{DEFAULT_ARCHIVE_NAME}',) - ) - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('collect_special_file_paths').and_return(('special',)) - create_command = ('borg', 'create', '--read-special') + REPO_ARCHIVE_WITH_PATHS - flexmock(module).should_receive('execute_command').with_args( - create_command + ('--dry-run', '--list'), - output_log_level=logging.INFO, - output_file=None, - borg_local_path='borg', - working_directory=None, - extra_environment=None, - ) - flexmock(module).should_receive('execute_command').with_args( - create_command + ('--exclude-from', '/excludes'), - output_log_level=logging.INFO, - output_file=None, - borg_local_path='borg', - working_directory=None, - extra_environment=None, - ) - - module.create_archive( - dry_run=False, - repository='repo', - location_config={ - 'source_directories': ['foo', 'bar'], - 'repositories': ['repo'], - 'read_special': True, - 'exclude_patterns': None, - }, - storage_config={}, - local_borg_version='1.2.3', - ) - - @pytest.mark.parametrize( 'option_name,option_value', (('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),), @@ -1911,6 +1855,131 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log ) +def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): + processes = flexmock() + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('map_directories_to_devices').and_return({}) + flexmock(module).should_receive('expand_directories').and_return(()) + flexmock(module).should_receive('pattern_root_directories').and_return([]) + flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) + flexmock(module).should_receive('expand_home_directories').and_return(()).and_return( + ('special',) + ) + flexmock(module).should_receive('write_pattern_file').and_return(None).and_return( + flexmock(name='/excludes') + ) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module).should_receive('ensure_files_readable') + flexmock(module).should_receive('make_pattern_flags').and_return(()) + flexmock(module).should_receive('make_exclude_flags').and_return(()).and_return( + '--exclude-from', '/excludes' + ) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('collect_special_file_paths').and_return(('special',)) + create_command = ( + 'borg', + 'create', + '--one-file-system', + '--read-special', + ) + REPO_ARCHIVE_WITH_PATHS + flexmock(module).should_receive('execute_command_with_processes').with_args( + create_command + ('--dry-run', '--list'), + processes=processes, + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=None, + ) + flexmock(module).should_receive('execute_command_with_processes').with_args( + create_command + ('--exclude-from', '/excludes'), + processes=processes, + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=None, + ) + + module.create_archive( + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + local_borg_version='1.2.3', + stream_processes=processes, + ) + + +def test_create_archive_with_stream_processes_and_read_special_does_not_add_special_files_to_excludes(): + processes = flexmock() + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('map_directories_to_devices').and_return({}) + flexmock(module).should_receive('expand_directories').and_return(()) + flexmock(module).should_receive('pattern_root_directories').and_return([]) + flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) + flexmock(module).should_receive('expand_home_directories').and_return(()).and_return( + ('special',) + ) + flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module).should_receive('ensure_files_readable') + flexmock(module).should_receive('make_pattern_flags').and_return(()) + flexmock(module).should_receive('make_exclude_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('collect_special_file_paths').and_return(('special',)) + create_command = ( + 'borg', + 'create', + '--one-file-system', + '--read-special', + ) + REPO_ARCHIVE_WITH_PATHS + flexmock(module).should_receive('execute_command_with_processes').with_args( + create_command + ('--dry-run', '--list'), + processes=processes, + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=None, + ) + flexmock(module).should_receive('execute_command_with_processes').with_args( + create_command, + processes=processes, + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=None, + ) + + module.create_archive( + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + 'read_special': True, + }, + storage_config={}, + local_borg_version='1.2.3', + stream_processes=processes, + ) + + def test_create_archive_with_json_calls_borg_with_json_parameter(): flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))