Browse Source

Leave exclude_patterns glob expansion to Borg, since doing it in borgmatic leads to confusing behavior (#132).

tags/1.2.15
Dan Helfman 1 year ago
parent
commit
6096a7181c
3 changed files with 89 additions and 38 deletions
  1. +2
    -0
      NEWS
  2. +14
    -1
      borgmatic/borg/create.py
  3. +73
    -37
      tests/unit/borg/test_create.py

+ 2
- 0
NEWS View File

@@ -1,5 +1,7 @@
1.2.15.dev0
* #127: Remove date echo from schema example, as it's not a substitute for real logging.
* #132: Leave exclude_patterns glob expansion to Borg, since doing it in borgmatic leads to
confusing behavior.
* #136: Handle and format validation errors raised during argument parsing.
* #139: Allow use of --stats flag when --create or --prune flags are implied.



+ 14
- 1
borgmatic/borg/create.py View File

@@ -32,6 +32,17 @@ def _expand_directories(directories):
)


def _expand_home_directories(directories):
'''
Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
Return the results as a tuple.
'''
if directories is None:
return ()

return tuple(os.path.expanduser(directory) for directory in directories)


def _write_pattern_file(patterns=None):
'''
Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -101,7 +112,9 @@ def create_archive(
sources = _expand_directories(location_config['source_directories'])

pattern_file = _write_pattern_file(location_config.get('patterns'))
exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
exclude_file = _write_pattern_file(
_expand_home_directories(location_config.get('exclude_patterns'))
)
checkpoint_interval = storage_config.get('checkpoint_interval', None)
chunker_params = storage_config.get('chunker_params', None)
compression = storage_config.get('compression', None)


+ 73
- 37
tests/unit/borg/test_create.py View File

@@ -43,6 +43,21 @@ def test_expand_directories_considers_none_as_no_directories():
assert paths == ()


def test_expand_home_directories_expands_tildes():
flexmock(module.os.path).should_receive('expanduser').with_args('~/bar').and_return('/foo/bar')
flexmock(module.os.path).should_receive('expanduser').with_args('baz').and_return('baz')

paths = module._expand_home_directories(('~/bar', 'baz'))

assert paths == ('/foo/bar', 'baz')


def test_expand_home_directories_considers_none_as_no_directories():
paths = module._expand_home_directories(None)

assert paths == ()


def test_write_pattern_file_does_not_raise():
temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None)
flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
@@ -149,7 +164,8 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo


def test_create_archive_calls_borg_with_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -169,7 +185,8 @@ def test_create_archive_calls_borg_with_parameters():

def test_create_archive_with_patterns_calls_borg_with_patterns():
pattern_flags = ('--patterns-from', 'patterns')
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(
flexmock(name='/tmp/patterns')
).and_return(None)
@@ -191,9 +208,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():

def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
exclude_flags = ('--exclude-from', 'excludes')
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(
('exclude',)
)
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
flexmock(name='/tmp/excludes')
)
@@ -214,7 +230,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():


def test_create_archive_with_log_info_calls_borg_with_info_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -235,7 +252,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():


def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -257,7 +275,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():


def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -277,9 +296,10 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():


def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_parameter():
""" --dry-run and --stats are mutually exclusive, see:
https://borgbackup.readthedocs.io/en/stable/usage/create.html#description """
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
# --dry-run and --stats are mutually exclusive, see:
# https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -300,9 +320,10 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param


def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_parameter():
""" --dry-run and --stats are mutually exclusive, see:
https://borgbackup.readthedocs.io/en/stable/usage/create.html#description """
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
# --dry-run and --stats are mutually exclusive, see:
# https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -325,7 +346,8 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para


def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -344,7 +366,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte


def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -363,7 +386,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param


def test_create_archive_with_compression_calls_borg_with_compression_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -382,7 +406,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(


def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -401,7 +426,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_


def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -421,7 +447,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par


def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -441,7 +468,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter


def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -461,7 +489,8 @@ def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parame


def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -481,7 +510,8 @@ def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_paramete


def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -501,7 +531,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(


def test_create_archive_with_local_path_calls_borg_via_local_path():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -521,7 +552,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():


def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -541,7 +573,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(


def test_create_archive_with_umask_calls_borg_with_umask_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -560,7 +593,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():


def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -579,7 +613,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():


def test_create_archive_with_json_calls_borg_with_json_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -599,9 +634,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():


def test_create_archive_with_source_directories_glob_expands():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(
()
)
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -623,7 +657,8 @@ def test_create_archive_with_source_directories_glob_expands():


def test_create_archive_with_non_matching_source_directories_glob_passes_through():
flexmock(module).should_receive('_expand_directories').and_return(('foo*',)).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -643,9 +678,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through


def test_create_archive_with_glob_calls_borg_with_expanded_directories():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(
()
)
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -666,7 +700,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():


def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -685,7 +720,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():


def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())


Loading…
Cancel
Save