From edb54b300bac377fda0e8a48747edea06cfd46b5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 20:11:49 -0700 Subject: [PATCH] Fixing up borg module to deal with new parsed config file structures. --- borgmatic/borg.py | 59 +++++++++++------ borgmatic/commands/borgmatic.py | 5 +- borgmatic/tests/unit/test_borg.py | 103 ++++++++++++++++-------------- 3 files changed, 95 insertions(+), 72 deletions(-) diff --git a/borgmatic/borg.py b/borgmatic/borg.py index f4c5ba9..22d3032 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -1,10 +1,11 @@ from datetime import datetime +import glob +import itertools import os -import re import platform +import re import subprocess -from glob import glob -from itertools import chain +import tempfile from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -22,18 +23,38 @@ def initialize(storage_config, command=COMMAND): os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase +def _write_exclude_file(exclude_patterns=None): + ''' + Given a sequence of exclude patterns, write them to a named temporary file and return it. Return + None if no patterns are provided. + ''' + if not exclude_patterns: + return None + + exclude_file = tempfile.NamedTemporaryFile('w') + exclude_file.write('\n'.join(exclude_patterns)) + exclude_file.flush() + + return exclude_file + + def create_archive( - excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND, - one_file_system=None, remote_path=None, + verbosity, storage_config, source_directories, repository, exclude_patterns=None, + command=COMMAND, one_file_system=None, remote_path=None, ): ''' - Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated - list of source directories, a local or remote repository path, and a command to run, create an - attic archive. + Given a vebosity flag, a storage config dict, a list of source directories, a local or remote + repository path, a list of exclude patterns, and a command to run, create an attic archive. ''' - sources = re.split('\s+', source_directories) - sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources)) - exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () + sources = tuple( + itertools.chain.from_iterable( + glob.glob(directory) or [directory] + for directory in source_directories + ) + ) + + exclude_file = _write_exclude_file(exclude_patterns) + exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () umask = storage_config.get('umask', None) @@ -109,12 +130,11 @@ DEFAULT_CHECKS = ('repository', 'archives') def _parse_checks(consistency_config): ''' - Given a consistency config with a space-separated "checks" option, transform it to a tuple of - named checks to run. + Given a consistency config with a "checks" list, transform it to a tuple of named checks to run. For example, given a retention config of: - {'checks': 'repository archives'} + {'checks': ['repository', 'archives']} This will be returned as: @@ -123,14 +143,11 @@ def _parse_checks(consistency_config): If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string "disabled", return an empty tuple, meaning that no checks should be run. ''' - checks = consistency_config.get('checks', '').strip() - if not checks: - return DEFAULT_CHECKS + checks = consistency_config.get('checks', []) + if checks == ['disabled']: + return () - return tuple( - check for check in consistency_config['checks'].split(' ') - if check.lower() not in ('disabled', '') - ) + return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS def _make_check_flags(checks, check_last=None): diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c2083d9..9bde8bc 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -48,10 +48,7 @@ def main(): # pragma: no cover remote_path = config.location['remote_path'] borg.initialize(config.storage) - # TODO: Use the new exclude_patterns. - borg.create_archive( - args.excludes_filename, args.verbosity, config.storage, **config.location - ) + borg.create_archive(args.verbosity, config.storage, **config.location) borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path) borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 3c2aa1c..57e75a3 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -30,6 +30,20 @@ def test_initialize_without_passphrase_should_not_set_environment(): finally: os.environ = orig_environ +def test_write_exclude_file_does_not_raise(): + temporary_file = flexmock( + name='filename', + write=lambda mode: None, + flush=lambda: None, + ) + flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file) + + module._write_exclude_file(['exclude']) + + +def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise(): + module._write_exclude_file([]) + def insert_subprocess_mock(check_call_command, **kwargs): subprocess = flexmock(STDOUT=STDOUT) @@ -53,110 +67,100 @@ def insert_datetime_mock(): ).mock -CREATE_COMMAND_WITHOUT_EXCLUDES = ('borg', 'create', 'repo::host-now', 'foo', 'bar') -CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes') +CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar') def test_create_archive_should_call_borg_with_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) -def test_create_archive_with_two_spaces_in_source_directories(): - insert_subprocess_mock(CREATE_COMMAND) +def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): + flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes')) + insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=['exclude'], verbosity=None, storage_config={}, - source_directories='foo bar', - repository='repo', - command='borg', - ) - - -def test_create_archive_with_none_excludes_filename_should_call_borg_without_excludes(): - insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - excludes_filename=None, - verbosity=None, - storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=VERBOSITY_SOME, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=VERBOSITY_LOTS, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_compression_should_call_borg_with_compression_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={'compression': 'rle'}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', one_file_system=True, @@ -164,15 +168,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', remote_path='borg1', @@ -180,63 +185,67 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={'umask': 740}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_source_directories_glob_expands(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() - flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) + flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( - excludes_filename=None, + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo*', + source_directories=['foo*'], repository='repo', command='borg', ) def test_create_archive_with_non_matching_source_directories_glob_passes_through(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*')) insert_platform_mock() insert_datetime_mock() - flexmock(module).should_receive('glob').with_args('foo*').and_return([]) + flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([]) module.create_archive( - excludes_filename=None, + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo*', + source_directories=['foo*'], repository='repo', command='borg', ) def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() - flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) + flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( - excludes_filename=None, + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo*', + source_directories=['foo*'], repository='repo', command='borg', ) @@ -329,7 +338,7 @@ def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parame def test_parse_checks_returns_them_as_tuple(): - checks = module._parse_checks({'checks': 'foo disabled bar'}) + checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']}) assert checks == ('foo', 'bar') @@ -341,13 +350,13 @@ def test_parse_checks_with_missing_value_returns_defaults(): def test_parse_checks_with_blank_value_returns_defaults(): - checks = module._parse_checks({'checks': ''}) + checks = module._parse_checks({'checks': []}) assert checks == module.DEFAULT_CHECKS def test_parse_checks_with_disabled_returns_no_checks(): - checks = module._parse_checks({'checks': 'disabled'}) + checks = module._parse_checks({'checks': ['disabled']}) assert checks == ()