Dropping support for Attic.

borgmatic/borg.py

@@ -0,0 +1,183 @@
+from datetime import datetime
+import os
+import re
+import platform
+import subprocess
+from glob import glob
+from itertools import chain
+
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+# Integration with Borg for actually handling backups.
+
+
+COMMAND = 'borg'
+
+
+def initialize(storage_config, command=COMMAND):
+    passphrase = storage_config.get('encryption_passphrase')
+
+    if passphrase:
+        os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase
+
+
+def create_archive(
+    excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND,
+    one_file_system=None
+):
+    '''
+    Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
+    list of source directories, a local or remote repository path, and a command to run, create an
+    attic archive.
+    '''
+    sources = re.split('\s+', source_directories)
+    sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources))
+    exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
+    compression = storage_config.get('compression', None)
+    compression_flags = ('--compression', compression) if compression else ()
+    umask = storage_config.get('umask', None)
+    umask_flags = ('--umask', str(umask)) if umask else ()
+    one_file_system_flags = ('--one-file-system',) if one_file_system else ()
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--stats',),
+        VERBOSITY_LOTS: ('--verbose', '--stats'),
+    }.get(verbosity, ())
+
+    full_command = (
+        command, 'create',
+        '{repo}::{hostname}-{timestamp}'.format(
+            repo=repository,
+            hostname=platform.node(),
+            timestamp=datetime.now().isoformat(),
+        ),
+    ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
+        umask_flags + verbosity_flags
+
+    subprocess.check_call(full_command)
+
+
+def _make_prune_flags(retention_config):
+    '''
+    Given a retention config dict mapping from option name to value, tranform it into an iterable of
+    command-line name-value flag pairs.
+
+    For example, given a retention config of:
+
+        {'keep_weekly': 4, 'keep_monthly': 6}
+
+    This will be returned as an iterable of:
+
+        (
+            ('--keep-weekly', '4'),
+            ('--keep-monthly', '6'),
+        )
+    '''
+    return (
+        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
+        for option_name, value in retention_config.items()
+    )
+
+
+def prune_archives(verbosity, repository, retention_config, command=COMMAND):
+    '''
+    Given a verbosity flag, a local or remote repository path, a retention config dict, and a
+    command to run, prune attic archives according the the retention policy specified in that
+    configuration.
+    '''
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--stats',),
+        VERBOSITY_LOTS: ('--verbose', '--stats'),
+    }.get(verbosity, ())
+
+    full_command = (
+        command, 'prune',
+        repository,
+    ) + tuple(
+        element
+        for pair in _make_prune_flags(retention_config)
+        for element in pair
+    ) + verbosity_flags
+
+    subprocess.check_call(full_command)
+
+
+DEFAULT_CHECKS = ('repository', 'archives')
+
+
+def _parse_checks(consistency_config):
+    '''
+    Given a consistency config with a space-separated "checks" option, transform it to a tuple of
+    named checks to run.
+
+    For example, given a retention config of:
+
+        {'checks': 'repository archives'}
+
+    This will be returned as:
+
+        ('repository', 'archives')
+
+    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
+    "disabled", return an empty tuple, meaning that no checks should be run.
+    '''
+    checks = consistency_config.get('checks', '').strip()
+    if not checks:
+        return DEFAULT_CHECKS
+
+    return tuple(
+        check for check in consistency_config['checks'].split(' ')
+        if check.lower() not in ('disabled', '')
+    )
+
+
+def _make_check_flags(checks, check_last=None):
+    '''
+    Given a parsed sequence of checks, transform it into tuple of command-line flags.
+
+    For example, given parsed checks of:
+
+        ('repository',)
+
+    This will be returned as:
+
+        ('--repository-only',)
+
+    Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
+    Borg supports this flag.
+    '''
+    last_flag = ('--last', check_last) if check_last else ()
+    if checks == DEFAULT_CHECKS:
+        return last_flag
+
+    return tuple(
+        '--{}-only'.format(check) for check in checks
+    ) + last_flag
+
+
+def check_archives(verbosity, repository, consistency_config, command=COMMAND):
+    '''
+    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
+    command to run, check the contained attic archives for consistency.
+
+    If there are no consistency checks to run, skip running them.
+    '''
+    checks = _parse_checks(consistency_config)
+    check_last = consistency_config.get('check_last', None)
+    if not checks:
+        return
+
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--verbose',),
+        VERBOSITY_LOTS: ('--verbose',),
+    }.get(verbosity, ())
+
+    full_command = (
+        command, 'check',
+        repository,
+    ) + _make_check_flags(checks, check_last) + verbosity_flags
+
+    # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
+    stdout = None if verbosity_flags else open(os.devnull, 'w')
+
+    subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)

borgmatic/command.py

@@ -0,0 +1,57 @@
+from __future__ import print_function
+from argparse import ArgumentParser
+import os
+from subprocess import CalledProcessError
+import sys
+
+from borgmatic import borg
+from borgmatic.config import parse_configuration, CONFIG_FORMAT
+
+
+DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'
+DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
+
+
+def parse_arguments(*arguments):
+    '''
+    Given the name of the command with which this script was invoked and command-line arguments,
+    parse the arguments and return them as an ArgumentParser instance. Use the command name to
+    determine the default configuration and excludes paths.
+    '''
+    parser = ArgumentParser()
+    parser.add_argument(
+        '-c', '--config',
+        dest='config_filename',
+        default=DEFAULT_CONFIG_FILENAME,
+        help='Configuration filename',
+    )
+    parser.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        default=DEFAULT_EXCLUDES_FILENAME if os.path.exists(DEFAULT_EXCLUDES_FILENAME) else None,
+        help='Excludes filename',
+    )
+    parser.add_argument(
+        '-v', '--verbosity',
+        type=int,
+        help='Display verbose progress (1 for some, 2 for lots)',
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():
+    try:
+        args = parse_arguments(*sys.argv[1:])
+        config = parse_configuration(args.config_filename, CONFIG_FORMAT)
+        repository = config.location['repository']
+
+        borg.initialize(config.storage)
+        borg.create_archive(
+            args.excludes_filename, args.verbosity, config.storage, **config.location
+        )
+        borg.prune_archives(args.verbosity, repository, config.retention)
+        borg.check_archives(args.verbosity, repository, config.consistency)
+    except (ValueError, IOError, CalledProcessError) as error:
+        print(error, file=sys.stderr)
+        sys.exit(1)

borgmatic/config.py

@@ -0,0 +1,166 @@
+from collections import OrderedDict, namedtuple
+
+try:
+    # Python 2
+    from ConfigParser import RawConfigParser
+except ImportError:
+    # Python 3
+    from configparser import RawConfigParser
+
+
+Section_format = namedtuple('Section_format', ('name', 'options'))
+Config_option = namedtuple('Config_option', ('name', 'value_type', 'required'))
+
+
+def option(name, value_type=str, required=True):
+    '''
+    Given a config file option name, an expected type for its value, and whether it's required,
+    return a Config_option capturing that information.
+    '''
+    return Config_option(name, value_type, required)
+
+
+CONFIG_FORMAT = (
+    Section_format(
+        'location',
+        (
+            option('source_directories'),
+            option('one_file_system', value_type=bool, required=False),
+            option('repository'),
+        ),
+    ),
+    Section_format(
+        'storage',
+        (
+            option('encryption_passphrase', required=False),
+            option('compression', required=False),
+            option('umask', required=False),
+        ),
+    ),
+    Section_format(
+        'retention',
+        (
+            option('keep_within', required=False),
+            option('keep_hourly', int, required=False),
+            option('keep_daily', int, required=False),
+            option('keep_weekly', int, required=False),
+            option('keep_monthly', int, required=False),
+            option('keep_yearly', int, required=False),
+            option('prefix', required=False),
+        ),
+    ),
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+            option('check_last', required=False),
+        ),
+    )
+)
+
+
+def validate_configuration_format(parser, config_format):
+    '''
+    Given an open RawConfigParser and an expected config file format, validate that the parsed
+    configuration file has the expected sections, that any required options are present in those
+    sections, and that there aren't any unexpected options.
+
+    A section is required if any of its contained options are required.
+
+    Raise ValueError if anything is awry.
+    '''
+    section_names = set(parser.sections())
+    required_section_names = tuple(
+        section.name for section in config_format
+        if any(option.required for option in section.options)
+    )
+
+    unknown_section_names = section_names - set(
+        section_format.name for section_format in config_format
+    )
+    if unknown_section_names:
+        raise ValueError(
+            'Unknown config sections found: {}'.format(', '.join(unknown_section_names))
+        )
+
+    missing_section_names = set(required_section_names) - section_names
+    if missing_section_names:
+        raise ValueError(
+            'Missing config sections: {}'.format(', '.join(missing_section_names))
+        )
+
+    for section_format in config_format:
+        if section_format.name not in section_names:
+            continue
+
+        option_names = parser.options(section_format.name)
+        expected_options = section_format.options
+
+        unexpected_option_names = set(option_names) - set(option.name for option in expected_options)
+
+        if unexpected_option_names:
+            raise ValueError(
+                'Unexpected options found in config section {}: {}'.format(
+                    section_format.name,
+                    ', '.join(sorted(unexpected_option_names)),
+                )
+            )
+
+        missing_option_names = tuple(
+            option.name for option in expected_options if option.required
+            if option.name not in option_names
+        )
+
+        if missing_option_names:
+            raise ValueError(
+                'Required options missing from config section {}: {}'.format(
+                    section_format.name,
+                    ', '.join(missing_option_names)
+                )
+            )
+
+
+def parse_section_options(parser, section_format):
+    '''
+    Given an open RawConfigParser and an expected section format, return the option values from that
+    section as a dict mapping from option name to value. Omit those options that are not present in
+    the parsed options.
+
+    Raise ValueError if any option values cannot be coerced to the expected Python data type.
+    '''
+    type_getter = {
+        str: parser.get,
+        int: parser.getint,
+        bool: parser.getboolean,
+    }
+
+    return OrderedDict(
+        (option.name, type_getter[option.value_type](section_format.name, option.name))
+        for option in section_format.options
+        if parser.has_option(section_format.name, option.name)
+    )
+
+
+def parse_configuration(config_filename, config_format):
+    '''
+    Given a config filename and an expected config file format, return the parsed configuration
+    as a namedtuple with one attribute for each parsed section.
+
+    Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
+    '''
+    parser = RawConfigParser()
+    if not parser.read(config_filename):
+        raise ValueError('Configuration file cannot be opened: {}'.format(config_filename))
+
+    validate_configuration_format(parser, config_format)
+
+    # Describes a parsed configuration, where each attribute is the name of a configuration file
+    # section and each value is a dict of that section's parsed options.
+    Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format))
+
+    return Parsed_config(
+        *(
+            parse_section_options(parser, section_format)
+            for section_format in config_format
+        )
+    )

borgmatic/tests/builtins.py

@@ -0,0 +1,11 @@
+from flexmock import flexmock
+import sys
+
+
+def builtins_mock():
+    try:
+        # Python 2
+        return flexmock(sys.modules['__builtin__'])
+    except KeyError:
+        # Python 3
+        return flexmock(sys.modules['builtins'])

borgmatic/tests/integration/test_command.py

@@ -0,0 +1,69 @@
+import os
+import sys
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic import command as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+    parser = module.parse_arguments()
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+    assert parser.verbosity == None
+
+
+def test_parse_arguments_with_filename_arguments_overrides_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+
+    assert parser.config_filename == 'myconfig'
+    assert parser.excludes_filename == 'myexcludes'
+    assert parser.verbosity == None
+
+
+def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
+    flexmock(os.path).should_receive('exists').and_return(False)
+
+    parser = module.parse_arguments()
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == None
+    assert parser.verbosity == None
+
+
+def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
+    flexmock(os.path).should_receive('exists').and_return(False)
+
+    parser = module.parse_arguments('--excludes', 'myexcludes')
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == 'myexcludes'
+    assert parser.verbosity == None
+
+
+def test_parse_arguments_with_verbosity_flag_overrides_default():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+    parser = module.parse_arguments('--verbosity', '1')
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+    assert parser.verbosity == 1
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(os.path).should_receive('exists').and_return(True)
+    original_stderr = sys.stderr
+    sys.stderr = sys.stdout
+
+    try:
+        with pytest.raises(SystemExit):
+            module.parse_arguments('--posix-me-harder')
+    finally:
+        sys.stderr = original_stderr

borgmatic/tests/integration/test_config.py

@@ -0,0 +1,29 @@
+try:
+    # Python 2
+    from cStringIO import StringIO
+except ImportError:
+    # Python 3
+    from io import StringIO
+
+from collections import OrderedDict
+import string
+
+from borgmatic import config as module
+
+
+def test_parse_section_options_with_punctuation_should_return_section_options():
+    parser = module.RawConfigParser()
+    parser.readfp(StringIO('[section]\nfoo: {}\n'.format(string.punctuation)))
+
+    section_format = module.Section_format(
+        'section',
+        (module.Config_option('foo', str, required=True),),
+    )
+
+    config = module.parse_section_options(parser, section_format)
+
+    assert config == OrderedDict(
+        (
+            ('foo', string.punctuation),
+        )
+    )

borgmatic/tests/integration/test_version.py

@@ -0,0 +1,8 @@
+import subprocess
+
+
+def test_setup_version_matches_news_version():
+    setup_version = subprocess.check_output(('python', 'setup.py', '--version')).decode('ascii')
+    news_version = open('NEWS').readline()
+
+    assert setup_version == news_version

borgmatic/tests/unit/test_borg.py

@@ -0,0 +1,420 @@
+from collections import OrderedDict
+from subprocess import STDOUT
+import os
+
+from flexmock import flexmock
+
+from borgmatic import borg as module
+from borgmatic.tests.builtins import builtins_mock
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def test_initialize_with_passphrase_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({'encryption_passphrase': 'pass'}, command='attic')
+        assert os.environ.get('ATTIC_PASSPHRASE') == 'pass'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_without_passphrase_should_not_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({}, command='attic')
+        assert os.environ.get('ATTIC_PASSPHRASE') == None
+    finally:
+        os.environ = orig_environ
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(STDOUT=STDOUT)
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+    flexmock(module).subprocess = subprocess
+
+
+def insert_subprocess_never():
+    subprocess = flexmock()
+    subprocess.should_receive('check_call').never()
+    flexmock(module).subprocess = subprocess
+
+
+def insert_platform_mock():
+    flexmock(module.platform).should_receive('node').and_return('host')
+
+
+def insert_datetime_mock():
+    flexmock(module).datetime = flexmock().should_receive('now').and_return(
+        flexmock().should_receive('isoformat').and_return('now').mock
+    ).mock
+
+
+CREATE_COMMAND_WITHOUT_EXCLUDES = ('attic', 'create', 'repo::host-now', 'foo', 'bar')
+CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes')
+
+
+def test_create_archive_should_call_attic_with_parameters():
+    insert_subprocess_mock(CREATE_COMMAND)
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_two_spaces_in_source_directories():
+    insert_subprocess_mock(CREATE_COMMAND)
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={},
+        source_directories='foo  bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes():
+    insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter():
+    insert_subprocess_mock(CREATE_COMMAND + ('--stats',))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=VERBOSITY_SOME,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_parameter():
+    insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=VERBOSITY_LOTS,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_compression_should_call_attic_with_compression_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={'compression': 'rle'},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_one_file_system_should_call_attic_with_one_file_system_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+        one_file_system=True,
+    )
+
+
+def test_create_archive_with_umask_should_call_attic_with_umask_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={'umask': 740},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_source_directories_glob_expands():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='foo*',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_non_matching_source_directories_glob_passes_through():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo*'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).should_receive('glob').with_args('foo*').and_return([])
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='foo*',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_glob_should_call_attic_with_expanded_directories():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='foo*',
+        repository='repo',
+        command='attic',
+    )
+
+
+BASE_PRUNE_FLAGS = (
+    ('--keep-daily', '1'),
+    ('--keep-weekly', '2'),
+    ('--keep-monthly', '3'),
+)
+
+
+def test_make_prune_flags_should_return_flags_from_config():
+    retention_config = OrderedDict(
+        (
+            ('keep_daily', 1),
+            ('keep_weekly', 2),
+            ('keep_monthly', 3),
+        )
+    )
+
+    result = module._make_prune_flags(retention_config)
+
+    assert tuple(result) == BASE_PRUNE_FLAGS
+
+
+PRUNE_COMMAND = (
+    'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
+)
+
+
+def test_prune_archives_should_call_attic_with_parameters():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND)
+
+    module.prune_archives(
+        verbosity=None,
+        repository='repo',
+        retention_config=retention_config,
+        command='attic',
+    )
+
+
+def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--stats',))
+
+    module.prune_archives(
+        repository='repo',
+        verbosity=VERBOSITY_SOME,
+        retention_config=retention_config,
+        command='attic',
+    )
+
+
+def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--verbose', '--stats',))
+
+    module.prune_archives(
+        repository='repo',
+        verbosity=VERBOSITY_LOTS,
+        retention_config=retention_config,
+        command='attic',
+    )
+
+
+def test_parse_checks_returns_them_as_tuple():
+    checks = module._parse_checks({'checks': 'foo disabled bar'})
+
+    assert checks == ('foo', 'bar')
+
+
+def test_parse_checks_with_missing_value_returns_defaults():
+    checks = module._parse_checks({})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_blank_value_returns_defaults():
+    checks = module._parse_checks({'checks': ''})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_disabled_returns_no_checks():
+    checks = module._parse_checks({'checks': 'disabled'})
+
+    assert checks == ()
+
+
+def test_make_check_flags_with_checks_returns_flags():
+    flags = module._make_check_flags(('foo', 'bar'))
+
+    assert flags == ('--foo-only', '--bar-only')
+
+
+def test_make_check_flags_with_default_checks_returns_no_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS)
+
+    assert flags == ()
+
+
+def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
+    flags = module._make_check_flags(('foo', 'bar'), check_last=3)
+
+    assert flags == ('--foo-only', '--bar-only', '--last', 3)
+
+
+def test_make_check_flags_with_last_returns_last_flag():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
+
+    assert flags == ('--last', 3)
+
+
+def test_check_archives_should_call_attic_with_parameters():
+    checks = flexmock()
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('attic', 'check', 'repo'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+    builtins_mock().should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        consistency_config=consistency_config,
+        command='attic',
+    )
+
+
+def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
+    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    insert_subprocess_mock(
+        ('attic', 'check', 'repo', '--verbose'),
+        stdout=None, stderr=STDOUT,
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.check_archives(
+        verbosity=VERBOSITY_SOME,
+        repository='repo',
+        consistency_config=consistency_config,
+        command='attic',
+    )
+
+
+def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
+    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    insert_subprocess_mock(
+        ('attic', 'check', 'repo', '--verbose'),
+        stdout=None, stderr=STDOUT,
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.check_archives(
+        verbosity=VERBOSITY_LOTS,
+        repository='repo',
+        consistency_config=consistency_config,
+        command='attic',
+    )
+
+
+def test_check_archives_without_any_checks_should_bail():
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
+    flexmock(module).should_receive('_parse_checks').and_return(())
+    insert_subprocess_never()
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        consistency_config=consistency_config,
+        command='attic',
+    )

borgmatic/tests/unit/test_config.py

@@ -0,0 +1,232 @@
+from collections import OrderedDict
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic import config as module
+
+
+def test_option_should_create_config_option():
+    option = module.option('name', bool, required=False)
+
+    assert option == module.Config_option('name', bool, False)
+
+
+def test_option_should_create_config_option_with_defaults():
+    option = module.option('name')
+
+    assert option == module.Config_option('name', str, True)
+
+
+def test_validate_configuration_format_with_valid_config_should_not_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section', 'other'))
+    parser.should_receive('options').with_args('section').and_return(('stuff',))
+    parser.should_receive('options').with_args('other').and_return(('such',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        module.Section_format(
+            'other',
+            options=(
+                module.Config_option('such', str, required=True),
+            ),
+        ),
+    )
+
+    module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_missing_required_section_should_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        # At least one option in this section is required, so the section is required.
+        module.Section_format(
+            'missing',
+            options=(
+                module.Config_option('such', str, required=False),
+                module.Config_option('things', str, required=True),
+            ),
+        ),
+    )
+
+    with pytest.raises(ValueError):
+        module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_missing_optional_section_should_not_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('stuff',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        # No options in the section are required, so the section is optional.
+        module.Section_format(
+            'missing',
+            options=(
+                module.Config_option('such', str, required=False),
+                module.Config_option('things', str, required=False),
+            ),
+        ),
+    )
+
+    module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_unknown_section_should_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section', 'extra'))
+    config_format = (
+        module.Section_format('section', options=()),
+    )
+
+    with pytest.raises(ValueError):
+        module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_missing_required_option_should_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('option',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('option', str, required=True),
+                module.Config_option('missing', str, required=True),
+            ),
+        ),
+    )
+
+    with pytest.raises(ValueError):
+        module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_missing_optional_option_should_not_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('option',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('option', str, required=True),
+                module.Config_option('missing', str, required=False),
+            ),
+        ),
+    )
+
+    module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_extra_option_should_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('option', 'extra'))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(module.Config_option('option', str, required=True),),
+        ),
+    )
+
+    with pytest.raises(ValueError):
+        module.validate_configuration_format(parser, config_format)
+
+
+def test_parse_section_options_should_return_section_options():
+    parser = flexmock()
+    parser.should_receive('get').with_args('section', 'foo').and_return('value')
+    parser.should_receive('getint').with_args('section', 'bar').and_return(1)
+    parser.should_receive('getboolean').never()
+    parser.should_receive('has_option').with_args('section', 'foo').and_return(True)
+    parser.should_receive('has_option').with_args('section', 'bar').and_return(True)
+
+    section_format = module.Section_format(
+        'section',
+        (
+            module.Config_option('foo', str, required=True),
+            module.Config_option('bar', int, required=True),
+        ),
+    )
+
+    config = module.parse_section_options(parser, section_format)
+
+    assert config == OrderedDict(
+        (
+            ('foo', 'value'),
+            ('bar', 1),
+        )
+    )
+
+
+def test_parse_section_options_for_missing_section_should_return_empty_dict():
+    parser = flexmock()
+    parser.should_receive('get').never()
+    parser.should_receive('getint').never()
+    parser.should_receive('getboolean').never()
+    parser.should_receive('has_option').with_args('section', 'foo').and_return(False)
+    parser.should_receive('has_option').with_args('section', 'bar').and_return(False)
+
+    section_format = module.Section_format(
+        'section',
+        (
+            module.Config_option('foo', str, required=False),
+            module.Config_option('bar', int, required=False),
+        ),
+    )
+
+    config = module.parse_section_options(parser, section_format)
+
+    assert config == OrderedDict()
+
+
+def insert_mock_parser():
+    parser = flexmock()
+    parser.should_receive('read').and_return([flexmock()])
+    module.RawConfigParser = lambda: parser
+
+    return parser
+
+
+def test_parse_configuration_should_return_section_configs():
+    parser = insert_mock_parser()
+    config_format = (flexmock(name='items'), flexmock(name='things'))
+    mock_module = flexmock(module)
+    mock_module.should_receive('validate_configuration_format').with_args(
+        parser, config_format,
+    ).once()
+    mock_section_configs = (flexmock(), flexmock())
+
+    for section_format, section_config in zip(config_format, mock_section_configs):
+        mock_module.should_receive('parse_section_options').with_args(
+            parser, section_format,
+        ).and_return(section_config).once()
+
+    parsed_config = module.parse_configuration('filename', config_format)
+
+    assert parsed_config == type(parsed_config)(*mock_section_configs)
+
+
+def test_parse_configuration_with_file_open_error_should_raise():
+    parser = insert_mock_parser()
+    parser.should_receive('read').and_return([])
+
+    with pytest.raises(ValueError):
+        module.parse_configuration('filename', config_format=flexmock())

borgmatic/verbosity.py

@@ -0,0 +1,2 @@
+VERBOSITY_SOME = 1
+VERBOSITY_LOTS = 2