diff --git a/NEWS b/NEWS index a1b2e490..1cdd1ee5 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,8 @@ 1.1.15.dev0 * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file. - * #55: Fix for missing tags/releases from Gitea and GitHub project hosting. * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together. + * #55: Fix for missing tags/releases from Gitea and GitHub project hosting. + * #58: Support for using tilde in exclude_patterns to reference home directory. 1.1.14 * #49: Fix for typo in --patterns-from option. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 04ffcba8..22c6c0b1 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -35,6 +35,22 @@ def _expand_directory(directory): return glob.glob(expanded_directory) or [expanded_directory] +def _expand_directories(directories): + ''' + Given a sequence of directory paths, expand tildes and globs in each one. Return all the + resulting directories as a single flattened tuple. + ''' + if directories is None: + return () + + return tuple( + itertools.chain.from_iterable( + _expand_directory(directory) + for directory in directories + ) + ) + + def _write_pattern_file(patterns=None): ''' Given a sequence of patterns, write them to a named temporary file and return it. Return None @@ -95,19 +111,14 @@ def create_archive( Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a storage config dict, create a Borg archive. ''' - sources = tuple( - itertools.chain.from_iterable( - _expand_directory(directory) - for directory in location_config['source_directories'] - ) - ) + sources = _expand_directories(location_config['source_directories']) pattern_file = _write_pattern_file(location_config.get('patterns')) pattern_flags = _make_pattern_flags( location_config, pattern_file.name if pattern_file else None, ) - exclude_file = _write_pattern_file(location_config.get('exclude_patterns')) + exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns'))) exclude_flags = _make_exclude_flags( location_config, exclude_file.name if exclude_file else None, diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3b2fcdd6..b91ce5d2 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -52,9 +52,9 @@ map: - type: scalar desc: | Any paths matching these patterns are included/excluded from backups. Globs are - expanded. Note that Borg considers this option experimental. See the output of - "borg help patterns" for more details. Quoting any value if it contains leading - punctuation, so it parses correctly. + expanded. (Tildes are not.) Note that Borg considers this option experimental. + See the output of "borg help patterns" for more details. Quote any value if it + contains leading punctuation, so it parses correctly. example: - 'R /' - '- /home/*/.cache' @@ -73,11 +73,11 @@ map: seq: - type: scalar desc: | - Any paths matching these patterns are excluded from backups. Globs are expanded. - See the output of "borg help patterns" for more details. + Any paths matching these patterns are excluded from backups. Globs and tildes + are expanded. See the output of "borg help patterns" for more details. example: - '*.pyc' - - /home/*/.cache + - ~/*/.cache - /etc/ssl exclude_from: seq: diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index 13325503..c0a54056 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -70,6 +70,21 @@ def test_expand_directory_with_glob_expands(): assert paths == ['foo', 'food'] +def test_expand_directories_flattens_expanded_directories(): + flexmock(module).should_receive('_expand_directory').with_args('~/foo').and_return(['/root/foo']) + flexmock(module).should_receive('_expand_directory').with_args('bar*').and_return(['bar', 'barf']) + + paths = module._expand_directories(('~/foo', 'bar*')) + + assert paths == ('/root/foo', 'bar', 'barf') + + +def test_expand_directories_considers_none_as_no_directories(): + paths = module._expand_directories(None) + + assert paths == () + + def test_write_pattern_file_does_not_raise(): temporary_file = flexmock( name='filename', @@ -194,7 +209,7 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo def test_create_archive_calls_borg_with_parameters(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -215,7 +230,7 @@ def test_create_archive_calls_borg_with_parameters(): def test_create_archive_with_patterns_calls_borg_with_patterns(): pattern_flags = ('--patterns-from', 'patterns') - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(flexmock(name='/tmp/patterns')).and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -236,7 +251,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns(): def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): exclude_flags = ('--exclude-from', 'excludes') - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(('exclude',)) flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(flexmock(name='/tmp/excludes')) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags) @@ -256,7 +271,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -277,7 +292,7 @@ def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter(): def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -297,7 +312,7 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter(): def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -318,7 +333,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): def test_create_archive_with_dry_run_and_verbosity_some_calls_borg_without_stats_parameter(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -339,7 +354,7 @@ def test_create_archive_with_dry_run_and_verbosity_some_calls_borg_without_stats def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats_parameter(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -360,7 +375,7 @@ def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats def test_create_archive_with_compression_calls_borg_with_compression_parameters(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -380,7 +395,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -400,7 +415,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -421,7 +436,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -442,7 +457,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( def test_create_archive_with_local_path_calls_borg_via_local_path(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -463,7 +478,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -484,7 +499,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( def test_create_archive_with_umask_calls_borg_with_umask_parameters(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -504,7 +519,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): def test_create_archive_with_source_directories_glob_expands(): - flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -525,7 +540,7 @@ def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_non_matching_source_directories_glob_passes_through(): - flexmock(module).should_receive('_expand_directory').and_return(['foo*']) + flexmock(module).should_receive('_expand_directories').and_return(('foo*',)).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -546,7 +561,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through def test_create_archive_with_glob_calls_borg_with_expanded_directories(): - flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -566,7 +581,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(()) @@ -588,7 +603,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): - flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return([]) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) flexmock(module).should_receive('_make_exclude_flags').and_return(())