From 16bebe983202a3bf15f250f585b3d906ae4e0f2e Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Thu, 30 Oct 2014 22:34:03 -0700
Subject: [PATCH 001/189] Initial import.

---
 .hgignore              |  3 +++
 README                 | 51 +++++++++++++++++++++++++++++++++++++
 atticmatic/__init__.py |  0
 atticmatic/attic.py    | 34 +++++++++++++++++++++++++
 atticmatic/command.py  | 38 ++++++++++++++++++++++++++++
 atticmatic/config.py   | 57 ++++++++++++++++++++++++++++++++++++++++++
 sample/atticmatic.cron |  3 +++
 sample/config          | 12 +++++++++
 sample/excludes        |  3 +++
 setup.py               | 11 ++++++++
 10 files changed, 212 insertions(+)
 create mode 100644 .hgignore
 create mode 100644 README
 create mode 100644 atticmatic/__init__.py
 create mode 100644 atticmatic/attic.py
 create mode 100644 atticmatic/command.py
 create mode 100644 atticmatic/config.py
 create mode 100644 sample/atticmatic.cron
 create mode 100644 sample/config
 create mode 100644 sample/excludes
 create mode 100644 setup.py

diff --git a/.hgignore b/.hgignore
new file mode 100644
index 000000000..e91865a0a
--- /dev/null
+++ b/.hgignore
@@ -0,0 +1,3 @@
+syntax: glob
+*.pyc
+*.egg-info
diff --git a/README b/README
new file mode 100644
index 000000000..5b61af7e5
--- /dev/null
+++ b/README
@@ -0,0 +1,51 @@
+Overview
+--------
+
+atticmatic is a simple Python wrapper script for the Attic backup software
+that initiates a backup and prunes any old backups according to a retention
+policy. The script supports specifying your settings in a declarative
+configuration file rather than having to put them all on the command-line, and
+handles common errors.
+
+Read more about Attic at https://attic-backup.org/
+
+
+Setup
+-----
+
+To get up and running with Attic, follow the Attic Quick Start guide at
+https://attic-backup.org/quickstart.html to create an Attic repository on a
+local or remote host.
+
+If the repository is on a remote host, make sure that your local root user has
+key-based ssh access to the desired user account on the remote host.
+
+To install atticmatic, run the following from the directory containing this
+README:
+
+    python setup.py install
+
+Then copy the following configuration files:
+
+    sudo cp sample/atticmatic.cron /etc/init.d/atticmatic
+    sudo cp sample/config sample/excludes /etc/atticmatic/
+
+Lastly, modify those files with your desired configuration.
+
+
+Usage
+-----
+
+You can run atticmatic and start a backup simply by invoking it without
+arguments:
+
+    atticmatic
+
+To get additional information about the progress of the backup, use the
+verbose option:
+
+    atticmattic --verbose
+
+If you'd like to see the available command-line arguments, view the help:
+
+    atticmattic --help
diff --git a/atticmatic/__init__.py b/atticmatic/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/atticmatic/attic.py b/atticmatic/attic.py
new file mode 100644
index 000000000..227490023
--- /dev/null
+++ b/atticmatic/attic.py
@@ -0,0 +1,34 @@
+from datetime import datetime
+
+import platform
+import subprocess
+
+
+def create_archive(excludes_filename, verbose, source_directories, repository):
+    sources = tuple(source_directories.split(' '))
+
+    command = (
+        'attic', 'create',
+        '--exclude-from', excludes_filename,
+        '{repo}::{hostname}-{timestamp}'.format(
+            repo=repository,
+            hostname=platform.node(),
+            timestamp=datetime.now().isoformat(),
+        ),
+    ) + sources + (
+        ('--verbose', '--stats') if verbose else ()
+    )
+
+    subprocess.check_call(command)
+
+
+def prune_archives(repository, verbose, keep_daily, keep_weekly, keep_monthly):
+    command = (
+        'attic', 'prune',
+        repository,
+        '--keep-daily', str(keep_daily),
+        '--keep-weekly', str(keep_weekly),
+        '--keep-monthly', str(keep_monthly),
+    ) + (('--verbose',) if verbose else ())
+
+    subprocess.check_call(command)
diff --git a/atticmatic/command.py b/atticmatic/command.py
new file mode 100644
index 000000000..ebfdda3d1
--- /dev/null
+++ b/atticmatic/command.py
@@ -0,0 +1,38 @@
+from __future__ import print_function
+from argparse import ArgumentParser
+from subprocess import CalledProcessError
+import sys
+
+from atticmatic.attic import create_archive, prune_archives
+from atticmatic.config import parse_configuration
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument(
+        '--config',
+        dest='config_filename',
+        default='/etc/atticmatic/config',
+        help='Configuration filename',
+    )
+    parser.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        default='/etc/atticmatic/excludes',
+        help='Excludes filename',
+    )
+    parser.add_argument(
+        '--verbose',
+        action='store_true',
+        help='Display verbose progress information',
+    )
+    args = parser.parse_args()
+
+    try:
+        location_config, retention_config = parse_configuration(args.config_filename)
+
+        create_archive(args.excludes_filename, args.verbose, *location_config)
+        prune_archives(location_config.repository, args.verbose, *retention_config)
+    except (ValueError, CalledProcessError), error:
+        print(error, file=sys.stderr)
+        sys.exit(1)
diff --git a/atticmatic/config.py b/atticmatic/config.py
new file mode 100644
index 000000000..7283ab85e
--- /dev/null
+++ b/atticmatic/config.py
@@ -0,0 +1,57 @@
+from collections import namedtuple
+from ConfigParser import SafeConfigParser
+
+
+CONFIG_SECTION_LOCATION = 'location'
+CONFIG_SECTION_RETENTION = 'retention'
+
+CONFIG_FORMAT = {
+    CONFIG_SECTION_LOCATION: ('source_directories', 'repository'),
+    CONFIG_SECTION_RETENTION: ('keep_daily', 'keep_weekly', 'keep_monthly'),
+}
+
+LocationConfig = namedtuple('LocationConfig', CONFIG_FORMAT[CONFIG_SECTION_LOCATION])
+RetentionConfig = namedtuple('RetentionConfig', CONFIG_FORMAT[CONFIG_SECTION_RETENTION])
+
+
+def parse_configuration(config_filename):
+    '''
+    Given a config filename of the expected format, return the parse configuration as a tuple of
+    (LocationConfig, RetentionConfig). Raise if the format is not as expected.
+    '''
+    parser = SafeConfigParser()
+    parser.read((config_filename,))
+    section_names = parser.sections()
+    expected_section_names = CONFIG_FORMAT.keys()
+
+    if set(section_names) != set(expected_section_names):
+        raise ValueError(
+            'Expected config sections {} but found sections: {}'.format(
+                ', '.join(expected_section_names),
+                ', '.join(section_names)
+            )
+        )
+
+    for section_name in section_names:
+        option_names = parser.options(section_name)
+        expected_option_names = CONFIG_FORMAT[section_name]
+
+        if set(option_names) != set(expected_option_names):
+            raise ValueError(
+                'Expected options {} in config section {} but found options: {}'.format(
+                    ', '.join(expected_option_names),
+                    section_name,
+                    ', '.join(option_names)
+                )
+            )
+
+    return (
+        LocationConfig(*(
+            parser.get(CONFIG_SECTION_LOCATION, option_name)
+            for option_name in CONFIG_FORMAT[CONFIG_SECTION_LOCATION]
+        )),
+        RetentionConfig(*(
+            parser.getint(CONFIG_SECTION_RETENTION, option_name)
+            for option_name in CONFIG_FORMAT[CONFIG_SECTION_RETENTION]
+        ))
+    )
diff --git a/sample/atticmatic.cron b/sample/atticmatic.cron
new file mode 100644
index 000000000..a38e382d8
--- /dev/null
+++ b/sample/atticmatic.cron
@@ -0,0 +1,3 @@
+# You can drop this file into /etc/cron.d/ to run atticmatic nightly.
+
+0 3 * * * root /usr/local/bin/atticmatic
diff --git a/sample/config b/sample/config
new file mode 100644
index 000000000..66f508d85
--- /dev/null
+++ b/sample/config
@@ -0,0 +1,12 @@
+[location]
+# Space-separated list of source directories to backup.
+source_directories: /home /etc
+
+# Path to local or remote Attic repository.
+repository: user@backupserver:sourcehostname.attic
+
+# Retention policy for how many backups to keep in each category.
+[retention]
+keep_daily: 7
+keep_weekly: 4
+keep_monthly: 6
diff --git a/sample/excludes b/sample/excludes
new file mode 100644
index 000000000..7e81c882a
--- /dev/null
+++ b/sample/excludes
@@ -0,0 +1,3 @@
+*.pyc
+/home/*/.cache
+/etc/ssl
diff --git a/setup.py b/setup.py
new file mode 100644
index 000000000..5ebfca315
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,11 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='atticmatic',
+    version='0.0.1',
+    description='A wrapper script for Attic backup software',
+    author='Dan Helfman',
+    author_email='witten@torsion.org',
+    packages=find_packages(),
+    entry_points={'console_scripts': ['atticmatic = atticmatic.command:main']},
+)

From 84922c7232b26b0df1e51f4ecde629dc4076e264 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 1 Nov 2014 17:46:04 -0700
Subject: [PATCH 002/189] Adding PATH necessary to find the attic binary.

---
 sample/atticmatic.cron | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sample/atticmatic.cron b/sample/atticmatic.cron
index a38e382d8..39bc6bcf0 100644
--- a/sample/atticmatic.cron
+++ b/sample/atticmatic.cron
@@ -1,3 +1,3 @@
 # You can drop this file into /etc/cron.d/ to run atticmatic nightly.
 
-0 3 * * * root /usr/local/bin/atticmatic
+0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/atticmatic

From db0f057b5413f63d3f4f1f8c27567b8ba9f6e8cf Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 17 Nov 2014 18:35:47 -0800
Subject: [PATCH 003/189] Adding contact info.

---
 README | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README b/README
index 5b61af7e5..0378ec5ba 100644
--- a/README
+++ b/README
@@ -49,3 +49,9 @@ verbose option:
 If you'd like to see the available command-line arguments, view the help:
 
     atticmattic --help
+
+
+Feedback
+--------
+
+Questions? Comments? Got a patch? Contact witten@torsion.org

From e567158246fb0d572fd9df4e5e558d8521bc81d4 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 17 Nov 2014 21:57:44 -0800
Subject: [PATCH 004/189] Adding unit tests for config module.

---
 README                               | 12 ++++
 atticmatic/tests/__init__.py         |  0
 atticmatic/tests/unit/__init__.py    |  0
 atticmatic/tests/unit/test_config.py | 83 ++++++++++++++++++++++++++++
 setup.py                             |  4 ++
 5 files changed, 99 insertions(+)
 create mode 100644 atticmatic/tests/__init__.py
 create mode 100644 atticmatic/tests/unit/__init__.py
 create mode 100644 atticmatic/tests/unit/test_config.py

diff --git a/README b/README
index 0378ec5ba..604a6b4b4 100644
--- a/README
+++ b/README
@@ -51,6 +51,18 @@ If you'd like to see the available command-line arguments, view the help:
     atticmattic --help
 
 
+Running tests
+-------------
+
+To install test-specific dependencies, first run:
+
+    python setup.py test
+
+To actually run tests, run:
+
+    nosetests --detailed-errors
+
+
 Feedback
 --------
 
diff --git a/atticmatic/tests/__init__.py b/atticmatic/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/atticmatic/tests/unit/__init__.py b/atticmatic/tests/unit/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
new file mode 100644
index 000000000..93cf509c6
--- /dev/null
+++ b/atticmatic/tests/unit/test_config.py
@@ -0,0 +1,83 @@
+from flexmock import flexmock
+from nose.tools import assert_raises
+
+from atticmatic import config as module
+
+
+def insert_mock_parser(section_names):
+    parser = flexmock()
+    parser.should_receive('read')
+    parser.should_receive('sections').and_return(section_names)
+    flexmock(module).SafeConfigParser = parser
+
+    return parser
+
+
+def test_parse_configuration_should_return_config_data():
+    section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION)
+    parser = insert_mock_parser(section_names)
+
+    for section_name in section_names:
+        parser.should_receive('options').with_args(section_name).and_return(
+            module.CONFIG_FORMAT[section_name],
+        )
+
+    expected_config = (
+        module.LocationConfig(flexmock(), flexmock()),
+        module.RetentionConfig(flexmock(), flexmock(), flexmock()),
+    )
+    sections = (
+        (module.CONFIG_SECTION_LOCATION, expected_config[0], 'get'),
+        (module.CONFIG_SECTION_RETENTION, expected_config[1], 'getint'),
+    )
+
+    for section_name, section_config, method_name in sections:
+        for index, option_name in enumerate(module.CONFIG_FORMAT[section_name]):
+            (
+                parser.should_receive(method_name).with_args(section_name, option_name)
+                .and_return(section_config[index])
+            )
+
+    config = module.parse_configuration(flexmock())
+
+    assert config == expected_config
+
+
+def test_parse_configuration_with_missing_section_should_raise():
+    insert_mock_parser((module.CONFIG_SECTION_LOCATION,))
+
+    with assert_raises(ValueError):
+        module.parse_configuration(flexmock())
+
+
+def test_parse_configuration_with_extra_section_should_raise():
+    insert_mock_parser((module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION, 'extra'))
+
+    with assert_raises(ValueError):
+        module.parse_configuration(flexmock())
+
+
+def test_parse_configuration_with_missing_option_should_raise():
+    section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION)
+    parser = insert_mock_parser(section_names)
+
+    for section_name in section_names:
+        parser.should_receive('options').with_args(section_name).and_return(
+            module.CONFIG_FORMAT[section_name][:-1],
+        )
+
+    with assert_raises(ValueError):
+        module.parse_configuration(flexmock())
+
+
+def test_parse_configuration_with_extra_option_should_raise():
+    section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION)
+    parser = insert_mock_parser(section_names)
+
+    for section_name in section_names:
+        parser.should_receive('options').with_args(section_name).and_return(
+            module.CONFIG_FORMAT[section_name] + ('extra',),
+        )
+
+    with assert_raises(ValueError):
+        module.parse_configuration(flexmock())
diff --git a/setup.py b/setup.py
index 5ebfca315..209db30cd 100644
--- a/setup.py
+++ b/setup.py
@@ -8,4 +8,8 @@ setup(
     author_email='witten@torsion.org',
     packages=find_packages(),
     entry_points={'console_scripts': ['atticmatic = atticmatic.command:main']},
+    tests_require=(
+        'flexmock',
+        'nose',
+    )
 )

From d1825097714c099a335609d7a3bf7c1afb7eace0 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 17 Nov 2014 22:19:34 -0800
Subject: [PATCH 005/189] Unit tests for attic invocation code.

---
 atticmatic/tests/unit/test_attic.py | 86 +++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 atticmatic/tests/unit/test_attic.py

diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py
new file mode 100644
index 000000000..9cce58766
--- /dev/null
+++ b/atticmatic/tests/unit/test_attic.py
@@ -0,0 +1,86 @@
+from flexmock import flexmock
+
+from atticmatic import attic as module
+
+
+def insert_subprocess_mock(check_call_command):
+    subprocess = flexmock()
+    subprocess.should_receive('check_call').with_args(check_call_command).once()
+    flexmock(module).subprocess = subprocess
+
+
+def insert_platform_mock():
+    flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock
+
+
+def insert_datetime_mock():
+    flexmock(module).datetime = flexmock().should_receive('now').and_return(
+        flexmock().should_receive('isoformat').and_return('now').mock
+    ).mock
+
+
+def test_create_archive_should_call_attic_with_parameters():
+    insert_subprocess_mock(
+        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbose=False,
+        source_directories='foo bar',
+        repository='repo',
+    )
+
+
+def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters():
+    insert_subprocess_mock(
+        (
+            'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar',
+            '--verbose', '--stats',
+        ),
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbose=True,
+        source_directories='foo bar',
+        repository='repo',
+    )
+
+
+def test_prune_archives_should_call_attic_with_parameters():
+    insert_subprocess_mock(
+        (
+            'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
+            '3',
+        ),
+    )
+
+    module.prune_archives(
+        repository='repo',
+        verbose=False,
+        keep_daily=1,
+        keep_weekly=2,
+        keep_monthly=3
+    )
+
+
+def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters():
+    insert_subprocess_mock(
+        (
+            'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
+            '3', '--verbose',
+        ),
+    )
+
+    module.prune_archives(
+        repository='repo',
+        verbose=True,
+        keep_daily=1,
+        keep_weekly=2,
+        keep_monthly=3
+    )

From 42d9e2bfd868eb2c79906cf6500aef94948dcc90 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 18 Nov 2014 18:22:51 -0800
Subject: [PATCH 006/189] Adding GPL v3 license.

---
 LICENSE | 675 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 675 insertions(+)
 create mode 100644 LICENSE

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 000000000..6b156fe1d
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,675 @@
+GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    {one line to give the program's name and a brief idea of what it does.}
+    Copyright (C) {year}  {name of author}
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    {project}  Copyright (C) {year}  {fullname}
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+

From cf4c2622262267c987bdfdf2802ea41221ae8e1f Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 18 Nov 2014 18:32:16 -0800
Subject: [PATCH 007/189] Note about hosting arrangement.

---
 README | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README b/README
index 604a6b4b4..9d4e7cb9e 100644
--- a/README
+++ b/README
@@ -9,6 +9,9 @@ handles common errors.
 
 Read more about Attic at https://attic-backup.org/
 
+atticmatic is hosted at http://torsion.org/hg/atticmatic/ and is mirrored on
+GitHub and BitBucket for convenience.
+
 
 Setup
 -----

From 200a1bd63e9e3ad9b6b1e406ff17a2f335aa9df7 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 25 Nov 2014 16:01:59 -0800
Subject: [PATCH 008/189] Updating README with clarifications and examples.

---
 README        | 24 ++++++++++++++++++++++--
 sample/config |  2 +-
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/README b/README
index 9d4e7cb9e..36301999b 100644
--- a/README
+++ b/README
@@ -7,6 +7,24 @@ policy. The script supports specifying your settings in a declarative
 configuration file rather than having to put them all on the command-line, and
 handles common errors.
 
+Here's an example config file:
+
+    [location]
+    # Space-separated list of source directories to backup.
+    source_directories: /home /etc
+
+    # Path to local or remote Attic repository.
+    repository: user@backupserver:sourcehostname.attic
+
+    [retention]
+    # Retention policy for how many backups to keep in each category.
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6
+
+Additionally, exclude patterns can be specified in a separate excludes config
+file, one pattern per line.
+
 Read more about Attic at https://attic-backup.org/
 
 atticmatic is hosted at http://torsion.org/hg/atticmatic/ and is mirrored on
@@ -44,8 +62,10 @@ arguments:
 
     atticmatic
 
-To get additional information about the progress of the backup, use the
-verbose option:
+This will also prune any old backups as per the configured retention policy.
+By default, the backup will proceed silently except in the case of errors. But
+if you'd like to to get additional information about the progress of the
+backup as it proceeds, use the verbose option instead:
 
     atticmattic --verbose
 
diff --git a/sample/config b/sample/config
index 66f508d85..3d7a111d5 100644
--- a/sample/config
+++ b/sample/config
@@ -5,8 +5,8 @@ source_directories: /home /etc
 # Path to local or remote Attic repository.
 repository: user@backupserver:sourcehostname.attic
 
-# Retention policy for how many backups to keep in each category.
 [retention]
+# Retention policy for how many backups to keep in each category.
 keep_daily: 7
 keep_weekly: 4
 keep_monthly: 6

From 704b97a636e7a98901ea12aee00b42a122cce3c1 Mon Sep 17 00:00:00 2001
From: Henning Schroder <henning.schroeder@gmail.com>
Date: Wed, 26 Nov 2014 13:04:14 +0100
Subject: [PATCH 009/189] fixed README: copy cronjob to /etc/cron.d instead of
 /etc/init.d (like comment in sample/atticmatic.cron correctly explains)

---
 README | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README b/README
index 36301999b..9a2a628f7 100644
--- a/README
+++ b/README
@@ -48,7 +48,7 @@ README:
 
 Then copy the following configuration files:
 
-    sudo cp sample/atticmatic.cron /etc/init.d/atticmatic
+    sudo cp sample/atticmatic.cron /etc/cron.d/atticmatic
     sudo cp sample/config sample/excludes /etc/atticmatic/
 
 Lastly, modify those files with your desired configuration.

From 10a449fe1af6fd6b1f977c4e22aed21c9e554251 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Wed, 26 Nov 2014 20:15:21 -0800
Subject: [PATCH 010/189] Adding note about making etc configuration directory
 before copying a file to it.

---
 README | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README b/README
index 9a2a628f7..ebdd062a9 100644
--- a/README
+++ b/README
@@ -49,6 +49,7 @@ README:
 Then copy the following configuration files:
 
     sudo cp sample/atticmatic.cron /etc/cron.d/atticmatic
+    sudo mkdir /etc/atticmatic/
     sudo cp sample/config sample/excludes /etc/atticmatic/
 
 Lastly, modify those files with your desired configuration.

From 5472424d5a3e1501584836d88f928c4cee68c3de Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Thu, 27 Nov 2014 09:29:31 -0800
Subject: [PATCH 011/189] Playing nicely with markdown.

---
 README | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/README b/README
index ebdd062a9..b30ebc1a2 100644
--- a/README
+++ b/README
@@ -1,5 +1,4 @@
-Overview
---------
+## Overview
 
 atticmatic is a simple Python wrapper script for the Attic backup software
 that initiates a backup and prunes any old backups according to a retention
@@ -31,8 +30,7 @@ atticmatic is hosted at http://torsion.org/hg/atticmatic/ and is mirrored on
 GitHub and BitBucket for convenience.
 
 
-Setup
------
+## Setup
 
 To get up and running with Attic, follow the Attic Quick Start guide at
 https://attic-backup.org/quickstart.html to create an Attic repository on a
@@ -55,8 +53,7 @@ Then copy the following configuration files:
 Lastly, modify those files with your desired configuration.
 
 
-Usage
------
+## Usage
 
 You can run atticmatic and start a backup simply by invoking it without
 arguments:
@@ -75,8 +72,7 @@ If you'd like to see the available command-line arguments, view the help:
     atticmattic --help
 
 
-Running tests
--------------
+## Running tests
 
 To install test-specific dependencies, first run:
 
@@ -87,7 +83,6 @@ To actually run tests, run:
     nosetests --detailed-errors
 
 
-Feedback
---------
+## Feedback
 
 Questions? Comments? Got a patch? Contact witten@torsion.org

From f862eda7d6db61007bd1c5e77a00364218ab45de Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Thu, 27 Nov 2014 09:34:13 -0800
Subject: [PATCH 012/189] Renaming README to indicate markdown.

---
 README => README.md | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename README => README.md (100%)

diff --git a/README b/README.md
similarity index 100%
rename from README
rename to README.md

From 814770c2a914c7b5389150bd67edfcf522eb8049 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 19:49:25 -0800
Subject: [PATCH 013/189] Markdown metadata and link formatting updates.

---
 README.md | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index b30ebc1a2..1b0f52532 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,13 @@
+title: Atticmatic
+date:
+
 ## Overview
 
-atticmatic is a simple Python wrapper script for the Attic backup software
-that initiates a backup and prunes any old backups according to a retention
-policy. The script supports specifying your settings in a declarative
-configuration file rather than having to put them all on the command-line, and
-handles common errors.
+atticmatic is a simple Python wrapper script for the [Attic backup
+software](https://attic-backup.org/) that initiates a backup and prunes any
+old backups according to a retention policy. The script supports specifying
+your settings in a declarative configuration file rather than having to put
+them all on the command-line, and handles common errors.
 
 Here's an example config file:
 
@@ -24,25 +27,23 @@ Here's an example config file:
 Additionally, exclude patterns can be specified in a separate excludes config
 file, one pattern per line.
 
-Read more about Attic at https://attic-backup.org/
-
-atticmatic is hosted at http://torsion.org/hg/atticmatic/ and is mirrored on
-GitHub and BitBucket for convenience.
+atticmatic is hosted at <https://torsion.org/atticmatic> and is mirrored on
+[GitHub](https://github.com/witten/atticmatic) and
+[BitBucket](https://bitbucket.org/dhelfman/atticmatic) for convenience.
 
 
 ## Setup
 
-To get up and running with Attic, follow the Attic Quick Start guide at
-https://attic-backup.org/quickstart.html to create an Attic repository on a
-local or remote host.
+To get up and running with Attic, follow the [Attic Quick
+Start](https://attic-backup.org/quickstart.html) guide to create an Attic
+repository on a local or remote host.
 
 If the repository is on a remote host, make sure that your local root user has
 key-based ssh access to the desired user account on the remote host.
 
-To install atticmatic, run the following from the directory containing this
-README:
+To install atticmatic, run the following command to download and install it:
 
-    python setup.py install
+    sudo pip install hg+https://torsion.org/hg/atticmatic
 
 Then copy the following configuration files:
 
@@ -85,4 +86,4 @@ To actually run tests, run:
 
 ## Feedback
 
-Questions? Comments? Got a patch? Contact witten@torsion.org
+Questions? Comments? Got a patch? Contact <mailto:witten@torsion.org>.

From 8a4167b7a3653b1910573d7132c307f82845b23a Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 20:15:21 -0800
Subject: [PATCH 014/189] Saving README when rendered such that it can be
 served easily.

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 1b0f52532..69aec5ac9 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,6 @@
 title: Atticmatic
 date:
+save_as: atticmatic/index.html
 
 ## Overview
 

From 65c837c828595d359cd8cf8285c5606511ef40ac Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 20:22:49 -0800
Subject: [PATCH 015/189] Mentioning source code location explicitly.

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 69aec5ac9..9e91077a9 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,8 @@ Here's an example config file:
 Additionally, exclude patterns can be specified in a separate excludes config
 file, one pattern per line.
 
-atticmatic is hosted at <https://torsion.org/atticmatic> and is mirrored on
+atticmatic is hosted at <https://torsion.org/atticmatic> with [source code
+available](https://torsion.org/hg/atticmatic). It's also mirrored on
 [GitHub](https://github.com/witten/atticmatic) and
 [BitBucket](https://bitbucket.org/dhelfman/atticmatic) for convenience.
 

From 45c65412668170c10463717ebca72b905caecc6e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 20:23:29 -0800
Subject: [PATCH 016/189] Python 3 compatible exceptions.

---
 atticmatic/command.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/atticmatic/command.py b/atticmatic/command.py
index ebfdda3d1..1466c3009 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -33,6 +33,6 @@ def main():
 
         create_archive(args.excludes_filename, args.verbose, *location_config)
         prune_archives(location_config.repository, args.verbose, *retention_config)
-    except (ValueError, CalledProcessError), error:
+    except (ValueError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

From 69971cd7e251246f1010e31eca8e84458240dac9 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 20:26:19 -0800
Subject: [PATCH 017/189] Python 3 ConfigParser compatibility.

---
 atticmatic/config.py                 | 4 ++--
 atticmatic/tests/unit/test_config.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/atticmatic/config.py b/atticmatic/config.py
index 7283ab85e..c11b784d4 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -1,5 +1,5 @@
 from collections import namedtuple
-from ConfigParser import SafeConfigParser
+from ConfigParser import ConfigParser
 
 
 CONFIG_SECTION_LOCATION = 'location'
@@ -19,7 +19,7 @@ def parse_configuration(config_filename):
     Given a config filename of the expected format, return the parse configuration as a tuple of
     (LocationConfig, RetentionConfig). Raise if the format is not as expected.
     '''
-    parser = SafeConfigParser()
+    parser = ConfigParser()
     parser.read((config_filename,))
     section_names = parser.sections()
     expected_section_names = CONFIG_FORMAT.keys()
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index 93cf509c6..818d3db4a 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -8,7 +8,7 @@ def insert_mock_parser(section_names):
     parser = flexmock()
     parser.should_receive('read')
     parser.should_receive('sections').and_return(section_names)
-    flexmock(module).SafeConfigParser = parser
+    flexmock(module).ConfigParser = parser
 
     return parser
 

From d0eae195565b60cbfe0194d96e08bc7e60c9b4b6 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 20:30:07 -0800
Subject: [PATCH 018/189] Adding authors/contributors file.

---
 AUTHORS | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 AUTHORS

diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 000000000..2a2f28fea
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,4 @@
+Dan Helfman <witten@torsion.org>: Main developer
+
+Alexander Görtz: Python 3 compatibility
+Henning Schroeder: Copy editing

From 126bb279cde133c65eda364da7dd660c22db926f Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 20:36:43 -0800
Subject: [PATCH 019/189] Expanding description.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 209db30cd..d23ba771b 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
     name='atticmatic',
     version='0.0.1',
-    description='A wrapper script for Attic backup software',
+    description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',
     packages=find_packages(),

From d46e37095000418d234146810e31ff57f25498dd Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 22:14:35 -0800
Subject: [PATCH 020/189] Fixing configparser import for Python 3.

---
 atticmatic/config.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/atticmatic/config.py b/atticmatic/config.py
index c11b784d4..20caa342e 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -1,5 +1,11 @@
 from collections import namedtuple
-from ConfigParser import ConfigParser
+
+try:
+    # Python 2
+    from ConfigParser import ConfigParser
+except ImportError:
+    # Python 3
+    from configparser import ConfigParser
 
 
 CONFIG_SECTION_LOCATION = 'location'

From 626dd66254b83d8b78b05926e1966961ae54a64f Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 22:35:25 -0800
Subject: [PATCH 021/189] Preventing ConfigParser from swallowing file read
 IOErrors, so that the user gets a more useful message.

---
 atticmatic/command.py                | 13 ++++++++++---
 atticmatic/config.py                 |  6 ++++--
 atticmatic/tests/unit/test_config.py |  3 ++-
 3 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/atticmatic/command.py b/atticmatic/command.py
index 1466c3009..7ea236b2e 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -7,7 +7,10 @@ from atticmatic.attic import create_archive, prune_archives
 from atticmatic.config import parse_configuration
 
 
-def main():
+def parse_arguments():
+    '''
+    Parse the command-line arguments from sys.argv and return them as an ArgumentParser instance.
+    '''
     parser = ArgumentParser()
     parser.add_argument(
         '--config',
@@ -26,13 +29,17 @@ def main():
         action='store_true',
         help='Display verbose progress information',
     )
-    args = parser.parse_args()
 
+    return parser.parse_args()
+
+
+def main():
     try:
+        args = parse_arguments()
         location_config, retention_config = parse_configuration(args.config_filename)
 
         create_archive(args.excludes_filename, args.verbose, *location_config)
         prune_archives(location_config.repository, args.verbose, *retention_config)
-    except (ValueError, CalledProcessError) as error:
+    except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/atticmatic/config.py b/atticmatic/config.py
index 20caa342e..ac8cad1ba 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -23,10 +23,12 @@ RetentionConfig = namedtuple('RetentionConfig', CONFIG_FORMAT[CONFIG_SECTION_RET
 def parse_configuration(config_filename):
     '''
     Given a config filename of the expected format, return the parse configuration as a tuple of
-    (LocationConfig, RetentionConfig). Raise if the format is not as expected.
+    (LocationConfig, RetentionConfig).
+
+    Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
     parser = ConfigParser()
-    parser.read((config_filename,))
+    parser.readfp(open(config_filename))
     section_names = parser.sections()
     expected_section_names = CONFIG_FORMAT.keys()
 
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index 818d3db4a..f393533b0 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -6,8 +6,9 @@ from atticmatic import config as module
 
 def insert_mock_parser(section_names):
     parser = flexmock()
-    parser.should_receive('read')
+    parser.should_receive('readfp')
     parser.should_receive('sections').and_return(section_names)
+    flexmock(module).open = lambda filename: None
     flexmock(module).ConfigParser = parser
 
     return parser

From 965dd1aabe0f56c3604b2489354cd131b2021fad Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 22:39:11 -0800
Subject: [PATCH 022/189] Adding sudo to installation of test dependencies, for
 consistency with installation of main dependencies.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9e91077a9..b02984b94 100644
--- a/README.md
+++ b/README.md
@@ -79,7 +79,7 @@ If you'd like to see the available command-line arguments, view the help:
 
 To install test-specific dependencies, first run:
 
-    python setup.py test
+    sudo python setup.py test
 
 To actually run tests, run:
 

From b94c106a36e5493884bc388b7e378b70b7cd3c6c Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 1 Dec 2014 22:47:51 -0800
Subject: [PATCH 023/189] For convenience, adding some short-form arguments in
 addition to the long-form arguments.

---
 atticmatic/command.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/atticmatic/command.py b/atticmatic/command.py
index 7ea236b2e..e9e8c9b52 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -13,7 +13,7 @@ def parse_arguments():
     '''
     parser = ArgumentParser()
     parser.add_argument(
-        '--config',
+        '-c', '--config',
         dest='config_filename',
         default='/etc/atticmatic/config',
         help='Configuration filename',
@@ -25,7 +25,7 @@ def parse_arguments():
         help='Excludes filename',
     )
     parser.add_argument(
-        '--verbose',
+        '-v', '--verbose',
         action='store_true',
         help='Display verbose progress information',
     )

From 056ed7184b60f0df457927cbcde08c0ab7940198 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 6 Dec 2014 18:35:20 -0800
Subject: [PATCH 024/189] Configuration support for additional attic prune
 flags: keep_within, keep_hourly, keep_yearly, and prefix.

---
 .hgignore                            |   3 +-
 NEWS                                 |   8 +
 atticmatic/attic.py                  |  39 ++++-
 atticmatic/command.py                |   4 +-
 atticmatic/config.py                 | 147 ++++++++++++------
 atticmatic/tests/unit/test_attic.py  |  41 ++++-
 atticmatic/tests/unit/test_config.py | 216 ++++++++++++++++++---------
 sample/config                        |   5 +
 setup.py                             |   2 +-
 9 files changed, 341 insertions(+), 124 deletions(-)
 create mode 100644 NEWS

diff --git a/.hgignore b/.hgignore
index e91865a0a..abab07771 100644
--- a/.hgignore
+++ b/.hgignore
@@ -1,3 +1,4 @@
 syntax: glob
-*.pyc
 *.egg-info
+*.pyc
+*.swp
diff --git a/NEWS b/NEWS
new file mode 100644
index 000000000..e9e6290d7
--- /dev/null
+++ b/NEWS
@@ -0,0 +1,8 @@
+0.0.2
+
+ * Configuration support for additional attic prune flags: keep_within, keep_hourly, keep_yearly,
+   and prefix.
+
+0.0.1
+
+ * Initial release.
diff --git a/atticmatic/attic.py b/atticmatic/attic.py
index 227490023..c8842103c 100644
--- a/atticmatic/attic.py
+++ b/atticmatic/attic.py
@@ -5,6 +5,10 @@ import subprocess
 
 
 def create_archive(excludes_filename, verbose, source_directories, repository):
+    '''
+    Given an excludes filename, a vebosity flag, a space-separated list of source directories, and
+    a local or remote repository path, create an attic archive.
+    '''
     sources = tuple(source_directories.split(' '))
 
     command = (
@@ -22,13 +26,40 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
     subprocess.check_call(command)
 
 
-def prune_archives(repository, verbose, keep_daily, keep_weekly, keep_monthly):
+def make_prune_flags(retention_config):
+    '''
+    Given a retention config dict mapping from option name to value, tranform it into an iterable of
+    command-line name-value flag pairs.
+
+    For example, given a retention config of:
+
+        {'keep_weekly': 4, 'keep_monthly': 6}
+
+    This will be returned as an iterable of:
+
+        (
+            ('--keep-weekly', '4'),
+            ('--keep-monthly', '6'),
+        )
+    '''
+    return (
+        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
+        for option_name, value in retention_config.items()
+    )
+
+
+def prune_archives(verbose, repository, retention_config):
+    '''
+    Given a verbosity flag, a local or remote repository path, and a retention config dict, prune
+    attic archives according the the retention policy specified in that configuration.
+    '''
     command = (
         'attic', 'prune',
         repository,
-        '--keep-daily', str(keep_daily),
-        '--keep-weekly', str(keep_weekly),
-        '--keep-monthly', str(keep_monthly),
+    ) + tuple(
+        element
+        for pair in make_prune_flags(retention_config)
+        for element in pair
     ) + (('--verbose',) if verbose else ())
 
     subprocess.check_call(command)
diff --git a/atticmatic/command.py b/atticmatic/command.py
index e9e8c9b52..d47f63a41 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -38,8 +38,8 @@ def main():
         args = parse_arguments()
         location_config, retention_config = parse_configuration(args.config_filename)
 
-        create_archive(args.excludes_filename, args.verbose, *location_config)
-        prune_archives(location_config.repository, args.verbose, *retention_config)
+        create_archive(args.excludes_filename, args.verbose, **location_config)
+        prune_archives(args.verbose, location_config['repository'], retention_config)
     except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/atticmatic/config.py b/atticmatic/config.py
index ac8cad1ba..f953860a9 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -1,4 +1,4 @@
-from collections import namedtuple
+from collections import OrderedDict, namedtuple
 
 try:
     # Python 2
@@ -8,58 +8,121 @@ except ImportError:
     from configparser import ConfigParser
 
 
-CONFIG_SECTION_LOCATION = 'location'
-CONFIG_SECTION_RETENTION = 'retention'
+Section_format = namedtuple('Section_format', ('name', 'options'))
+Config_option = namedtuple('Config_option', ('name', 'value_type', 'required'))
 
-CONFIG_FORMAT = {
-    CONFIG_SECTION_LOCATION: ('source_directories', 'repository'),
-    CONFIG_SECTION_RETENTION: ('keep_daily', 'keep_weekly', 'keep_monthly'),
-}
 
-LocationConfig = namedtuple('LocationConfig', CONFIG_FORMAT[CONFIG_SECTION_LOCATION])
-RetentionConfig = namedtuple('RetentionConfig', CONFIG_FORMAT[CONFIG_SECTION_RETENTION])
+def option(name, value_type=str, required=True):
+    '''
+    Given a config file option name, an expected type for its value, and whether it's required,
+    return a Config_option capturing that information.
+    '''
+    return Config_option(name, value_type, required)
+
+
+CONFIG_FORMAT = (
+    Section_format(
+        'location',
+        (
+            option('source_directories'),
+            option('repository'),
+        ),
+    ),
+    Section_format(
+        'retention',
+        (
+            option('keep_within', required=False),
+            option('keep_hourly', int, required=False),
+            option('keep_daily', int, required=False),
+            option('keep_weekly', int, required=False),
+            option('keep_monthly', int, required=False),
+            option('keep_yearly', int, required=False),
+            option('prefix', required=False),
+        ),
+    )
+)
+
+
+def validate_configuration_format(parser, config_format):
+    '''
+    Given an open ConfigParser and an expected config file format, validate that the parsed
+    configuration file has the expected sections, that any required options are present in those
+    sections, and that there aren't any unexpected options.
+
+    Raise ValueError if anything is awry.
+    '''
+    section_names = parser.sections()
+    required_section_names = tuple(section.name for section in config_format)
+
+    if set(section_names) != set(required_section_names):
+        raise ValueError(
+            'Expected config sections {} but found sections: {}'.format(
+                ', '.join(required_section_names),
+                ', '.join(section_names)
+            )
+        )
+
+    for section_format in config_format:
+        option_names = parser.options(section_format.name)
+        expected_options = section_format.options
+
+        unexpected_option_names = set(option_names) - set(option.name for option in expected_options)
+
+        if unexpected_option_names:
+            raise ValueError(
+                'Unexpected options found in config section {}: {}'.format(
+                    section_format.name,
+                    ', '.join(sorted(unexpected_option_names)),
+                )
+            )
+
+        missing_option_names = tuple(
+            option.name for option in expected_options if option.required
+            if option.name not in option_names
+        )
+
+        if missing_option_names:
+            raise ValueError(
+                'Required options missing from config section {}: {}'.format(
+                    section_format.name,
+                    ', '.join(missing_option_names)
+                )
+            )
+
+
+def parse_section_options(parser, section_format):
+    '''
+    Given an open ConfigParser and an expected section format, return the option values from that
+    section as a dict mapping from option name to value. Omit those options that are not present in
+    the parsed options.
+
+    Raise ValueError if any option values cannot be coerced to the expected Python data type.
+    '''
+    type_getter = {
+        str: parser.get,
+        int: parser.getint,
+    }
+
+    return OrderedDict(
+        (option.name, type_getter[option.value_type](section_format.name, option.name))
+        for option in section_format.options
+        if parser.has_option(section_format.name, option.name)
+    )
 
 
 def parse_configuration(config_filename):
     '''
-    Given a config filename of the expected format, return the parse configuration as a tuple of
-    (LocationConfig, RetentionConfig).
+    Given a config filename of the expected format, return the parsed configuration as a tuple of
+    (location config, retention config) where each config is a dict of that section's options.
 
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
     parser = ConfigParser()
     parser.readfp(open(config_filename))
-    section_names = parser.sections()
-    expected_section_names = CONFIG_FORMAT.keys()
 
-    if set(section_names) != set(expected_section_names):
-        raise ValueError(
-            'Expected config sections {} but found sections: {}'.format(
-                ', '.join(expected_section_names),
-                ', '.join(section_names)
-            )
-        )
+    validate_configuration_format(parser, CONFIG_FORMAT)
 
-    for section_name in section_names:
-        option_names = parser.options(section_name)
-        expected_option_names = CONFIG_FORMAT[section_name]
-
-        if set(option_names) != set(expected_option_names):
-            raise ValueError(
-                'Expected options {} in config section {} but found options: {}'.format(
-                    ', '.join(expected_option_names),
-                    section_name,
-                    ', '.join(option_names)
-                )
-            )
-
-    return (
-        LocationConfig(*(
-            parser.get(CONFIG_SECTION_LOCATION, option_name)
-            for option_name in CONFIG_FORMAT[CONFIG_SECTION_LOCATION]
-        )),
-        RetentionConfig(*(
-            parser.getint(CONFIG_SECTION_RETENTION, option_name)
-            for option_name in CONFIG_FORMAT[CONFIG_SECTION_RETENTION]
-        ))
+    return tuple(
+        parse_section_options(parser, section_format)
+        for section_format in CONFIG_FORMAT
     )
diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py
index 9cce58766..44b38bbd0 100644
--- a/atticmatic/tests/unit/test_attic.py
+++ b/atticmatic/tests/unit/test_attic.py
@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from flexmock import flexmock
 
 from atticmatic import attic as module
@@ -52,7 +54,32 @@ def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters()
     )
 
 
+BASE_PRUNE_FLAGS = (
+    ('--keep-daily', '1'),
+    ('--keep-weekly', '2'),
+    ('--keep-monthly', '3'),
+)
+
+
+def test_make_prune_flags_should_return_flags_from_config():
+    retention_config = OrderedDict(
+        (
+            ('keep_daily', 1),
+            ('keep_weekly', 2),
+            ('keep_monthly', 3),
+        )
+    )
+
+    result = module.make_prune_flags(retention_config)
+
+    assert tuple(result) == BASE_PRUNE_FLAGS
+
+
 def test_prune_archives_should_call_attic_with_parameters():
+    retention_config = flexmock()
+    flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
     insert_subprocess_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
@@ -61,15 +88,17 @@ def test_prune_archives_should_call_attic_with_parameters():
     )
 
     module.prune_archives(
-        repository='repo',
         verbose=False,
-        keep_daily=1,
-        keep_weekly=2,
-        keep_monthly=3
+        repository='repo',
+        retention_config=retention_config,
     )
 
 
 def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters():
+    retention_config = flexmock()
+    flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
     insert_subprocess_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
@@ -80,7 +109,5 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
     module.prune_archives(
         repository='repo',
         verbose=True,
-        keep_daily=1,
-        keep_weekly=2,
-        keep_monthly=3
+        retention_config=retention_config,
     )
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index f393533b0..0576dc8f6 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -1,84 +1,166 @@
+from collections import OrderedDict
+
 from flexmock import flexmock
 from nose.tools import assert_raises
 
 from atticmatic import config as module
 
 
-def insert_mock_parser(section_names):
+def test_option_should_create_config_option():
+    option = module.option('name', bool, required=False)
+
+    assert option == module.Config_option('name', bool, False)
+
+
+def test_option_should_create_config_option_with_defaults():
+    option = module.option('name')
+
+    assert option == module.Config_option('name', str, True)
+
+
+def test_validate_configuration_format_with_valid_config_should_not_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section', 'other'))
+    parser.should_receive('options').with_args('section').and_return(('stuff',))
+    parser.should_receive('options').with_args('other').and_return(('such',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        module.Section_format(
+            'other',
+            options=(
+                module.Config_option('such', str, required=True),
+            ),
+        ),
+    )
+
+    module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_missing_section_should_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    config_format = (
+        module.Section_format('section', options=()),
+        module.Section_format('missing', options=()),
+    )
+
+    with assert_raises(ValueError):
+        module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_extra_section_should_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section', 'extra'))
+    config_format = (
+        module.Section_format('section', options=()),
+    )
+
+    with assert_raises(ValueError):
+        module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_missing_required_option_should_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('option',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('option', str, required=True),
+                module.Config_option('missing', str, required=True),
+            ),
+        ),
+    )
+
+    with assert_raises(ValueError):
+        module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_missing_optional_option_should_not_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('option',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('option', str, required=True),
+                module.Config_option('missing', str, required=False),
+            ),
+        ),
+    )
+
+    module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_extra_option_should_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('option', 'extra'))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(module.Config_option('option', str, required=True),),
+        ),
+    )
+
+    with assert_raises(ValueError):
+        module.validate_configuration_format(parser, config_format)
+
+
+def test_parse_section_options_should_return_section_options():
+    parser = flexmock()
+    parser.should_receive('get').with_args('section', 'foo').and_return('value')
+    parser.should_receive('getint').with_args('section', 'bar').and_return(1)
+    parser.should_receive('has_option').with_args('section', 'foo').and_return(True)
+    parser.should_receive('has_option').with_args('section', 'bar').and_return(True)
+
+    section_format = module.Section_format(
+        'section',
+        (
+            module.Config_option('foo', str, required=True),
+            module.Config_option('bar', int, required=True),
+        ),
+    )
+
+    config = module.parse_section_options(parser, section_format)
+
+    assert config == OrderedDict(
+        (
+            ('foo', 'value'),
+            ('bar', 1),
+        )
+    )
+
+
+def insert_mock_parser():
     parser = flexmock()
     parser.should_receive('readfp')
-    parser.should_receive('sections').and_return(section_names)
     flexmock(module).open = lambda filename: None
     flexmock(module).ConfigParser = parser
 
     return parser
 
 
-def test_parse_configuration_should_return_config_data():
-    section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION)
-    parser = insert_mock_parser(section_names)
+def test_parse_configuration_should_return_section_configs():
+    parser = insert_mock_parser()
+    mock_module = flexmock(module)
+    mock_module.should_receive('validate_configuration_format').with_args(
+        parser, module.CONFIG_FORMAT,
+    ).once()
+    mock_section_configs = (flexmock(), flexmock())
 
-    for section_name in section_names:
-        parser.should_receive('options').with_args(section_name).and_return(
-            module.CONFIG_FORMAT[section_name],
-        )
+    for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
+        mock_module.should_receive('parse_section_options').with_args(
+            parser, section_format,
+        ).and_return(section_config).once()
 
-    expected_config = (
-        module.LocationConfig(flexmock(), flexmock()),
-        module.RetentionConfig(flexmock(), flexmock(), flexmock()),
-    )
-    sections = (
-        (module.CONFIG_SECTION_LOCATION, expected_config[0], 'get'),
-        (module.CONFIG_SECTION_RETENTION, expected_config[1], 'getint'),
-    )
+    section_configs = module.parse_configuration('filename')
 
-    for section_name, section_config, method_name in sections:
-        for index, option_name in enumerate(module.CONFIG_FORMAT[section_name]):
-            (
-                parser.should_receive(method_name).with_args(section_name, option_name)
-                .and_return(section_config[index])
-            )
-
-    config = module.parse_configuration(flexmock())
-
-    assert config == expected_config
-
-
-def test_parse_configuration_with_missing_section_should_raise():
-    insert_mock_parser((module.CONFIG_SECTION_LOCATION,))
-
-    with assert_raises(ValueError):
-        module.parse_configuration(flexmock())
-
-
-def test_parse_configuration_with_extra_section_should_raise():
-    insert_mock_parser((module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION, 'extra'))
-
-    with assert_raises(ValueError):
-        module.parse_configuration(flexmock())
-
-
-def test_parse_configuration_with_missing_option_should_raise():
-    section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION)
-    parser = insert_mock_parser(section_names)
-
-    for section_name in section_names:
-        parser.should_receive('options').with_args(section_name).and_return(
-            module.CONFIG_FORMAT[section_name][:-1],
-        )
-
-    with assert_raises(ValueError):
-        module.parse_configuration(flexmock())
-
-
-def test_parse_configuration_with_extra_option_should_raise():
-    section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION)
-    parser = insert_mock_parser(section_names)
-
-    for section_name in section_names:
-        parser.should_receive('options').with_args(section_name).and_return(
-            module.CONFIG_FORMAT[section_name] + ('extra',),
-        )
-
-    with assert_raises(ValueError):
-        module.parse_configuration(flexmock())
+    assert section_configs == mock_section_configs
diff --git a/sample/config b/sample/config
index 3d7a111d5..aa2acf3f5 100644
--- a/sample/config
+++ b/sample/config
@@ -7,6 +7,11 @@ repository: user@backupserver:sourcehostname.attic
 
 [retention]
 # Retention policy for how many backups to keep in each category.
+# See https://attic-backup.org/usage.html#attic-prune for details.
+#keep_within: 3h
+#keep_hourly: 24
 keep_daily: 7
 keep_weekly: 4
 keep_monthly: 6
+keep_yearly: 1
+#prefix: sourcehostname
diff --git a/setup.py b/setup.py
index d23ba771b..e73711233 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.1',
+    version='0.0.2',
     description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

From 18267b96771b2a64638e8dc1f8dbd1da3eca3bcd Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 6 Dec 2014 18:35:28 -0800
Subject: [PATCH 025/189] Added tag 0.0.2 for changeset 467d3a3ce918

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .hgtags

diff --git a/.hgtags b/.hgtags
new file mode 100644
index 000000000..ec58244b8
--- /dev/null
+++ b/.hgtags
@@ -0,0 +1 @@
+467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2

From 511314a54a50d371de7c358d1e6e8a337eb47875 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 20 Dec 2014 10:56:03 -0800
Subject: [PATCH 026/189] Adding a note about repository encryption.

---
 README.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b02984b94..034bbfecc 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,12 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on
 
 To get up and running with Attic, follow the [Attic Quick
 Start](https://attic-backup.org/quickstart.html) guide to create an Attic
-repository on a local or remote host.
+repository on a local or remote host. Note that if you plan to run atticmatic
+on a schedule with cron, and you encrypt your attic repository with a
+passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE`
+environment variable. See [attic's repository encryption
+documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
+more info.
 
 If the repository is on a remote host, make sure that your local root user has
 key-based ssh access to the desired user account on the remote host.

From dbd312981e4b2600905075ef8def4d1b56b59a02 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 20 Dec 2014 11:37:25 -0800
Subject: [PATCH 027/189] Integration tests for argument parsing.

---
 NEWS                                         |  5 +++
 atticmatic/command.py                        | 14 ++++---
 atticmatic/tests/integration/__init__.py     |  0
 atticmatic/tests/integration/test_command.py | 40 ++++++++++++++++++++
 4 files changed, 54 insertions(+), 5 deletions(-)
 create mode 100644 atticmatic/tests/integration/__init__.py
 create mode 100644 atticmatic/tests/integration/test_command.py

diff --git a/NEWS b/NEWS
index e9e6290d7..17bf8d3f1 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+default
+
+ * Integration tests for argument parsing.
+ * Documentation updates about repository encryption.
+
 0.0.2
 
  * Configuration support for additional attic prune flags: keep_within, keep_hourly, keep_yearly,
diff --git a/atticmatic/command.py b/atticmatic/command.py
index d47f63a41..4623c47b9 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -7,7 +7,11 @@ from atticmatic.attic import create_archive, prune_archives
 from atticmatic.config import parse_configuration
 
 
-def parse_arguments():
+DEFAULT_CONFIG_FILENAME = '/etc/atticmatic/config'
+DEFAULT_EXCLUDES_FILENAME = '/etc/atticmatic/excludes'
+
+
+def parse_arguments(*arguments):
     '''
     Parse the command-line arguments from sys.argv and return them as an ArgumentParser instance.
     '''
@@ -15,13 +19,13 @@ def parse_arguments():
     parser.add_argument(
         '-c', '--config',
         dest='config_filename',
-        default='/etc/atticmatic/config',
+        default=DEFAULT_CONFIG_FILENAME,
         help='Configuration filename',
     )
     parser.add_argument(
         '--excludes',
         dest='excludes_filename',
-        default='/etc/atticmatic/excludes',
+        default=DEFAULT_EXCLUDES_FILENAME,
         help='Excludes filename',
     )
     parser.add_argument(
@@ -30,12 +34,12 @@ def parse_arguments():
         help='Display verbose progress information',
     )
 
-    return parser.parse_args()
+    return parser.parse_args(arguments)
 
 
 def main():
     try:
-        args = parse_arguments()
+        args = parse_arguments(*sys.argv[1:])
         location_config, retention_config = parse_configuration(args.config_filename)
 
         create_archive(args.excludes_filename, args.verbose, **location_config)
diff --git a/atticmatic/tests/integration/__init__.py b/atticmatic/tests/integration/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py
new file mode 100644
index 000000000..3a5e7943d
--- /dev/null
+++ b/atticmatic/tests/integration/test_command.py
@@ -0,0 +1,40 @@
+import sys
+
+from nose.tools import assert_raises
+
+from atticmatic import command as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    parser = module.parse_arguments()
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+    assert parser.verbose == False
+
+
+def test_parse_arguments_with_filename_arguments_overrides_defaults():
+    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+
+    assert parser.config_filename == 'myconfig'
+    assert parser.excludes_filename == 'myexcludes'
+    assert parser.verbose == False
+
+
+def test_parse_arguments_with_verbose_flag_overrides_default():
+    parser = module.parse_arguments('--verbose')
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+    assert parser.verbose == True
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    original_stderr = sys.stderr
+    sys.stderr = sys.stdout
+
+    try:
+        with assert_raises(SystemExit):
+            module.parse_arguments('--posix-me-harder')
+    finally:
+        sys.stderr = original_stderr

From b1113d57ae96db68805edc045c9c21e63a253d8d Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 20 Dec 2014 11:42:27 -0800
Subject: [PATCH 028/189] Correcting doc string based on updated command-line
 arguments source.

---
 atticmatic/command.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/atticmatic/command.py b/atticmatic/command.py
index 4623c47b9..ff31ca786 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -13,7 +13,7 @@ DEFAULT_EXCLUDES_FILENAME = '/etc/atticmatic/excludes'
 
 def parse_arguments(*arguments):
     '''
-    Parse the command-line arguments from sys.argv and return them as an ArgumentParser instance.
+    Parse the given command-line arguments and return them as an ArgumentParser instance.
     '''
     parser = ArgumentParser()
     parser.add_argument(

From eaf2bd22c1eabf00749d3ef6c875e4b47adc1a28 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 14 Feb 2015 09:23:40 -0800
Subject: [PATCH 029/189] After pruning, run attic's consistency checks on all
 archives.

---
 NEWS                                |  3 ++-
 README.md                           | 13 ++++++-----
 atticmatic/attic.py                 | 18 ++++++++++++++-
 atticmatic/command.py               |  6 +++--
 atticmatic/tests/unit/test_attic.py | 35 +++++++++++++++++++++++++++--
 5 files changed, 64 insertions(+), 11 deletions(-)

diff --git a/NEWS b/NEWS
index 17bf8d3f1..93d33a5d6 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
-default
+0.0.3
 
+ * After pruning, run attic's consistency checks on all archives.
  * Integration tests for argument parsing.
  * Documentation updates about repository encryption.
 
diff --git a/README.md b/README.md
index 034bbfecc..4a2d2110c 100644
--- a/README.md
+++ b/README.md
@@ -5,10 +5,11 @@ save_as: atticmatic/index.html
 ## Overview
 
 atticmatic is a simple Python wrapper script for the [Attic backup
-software](https://attic-backup.org/) that initiates a backup and prunes any
-old backups according to a retention policy. The script supports specifying
-your settings in a declarative configuration file rather than having to put
-them all on the command-line, and handles common errors.
+software](https://attic-backup.org/) that initiates a backup, prunes any old
+backups according to a retention policy, and validates backups for
+consistency. The script supports specifying your settings in a declarative
+configuration file rather than having to put them all on the command-line, and
+handles common errors.
 
 Here's an example config file:
 
@@ -68,7 +69,9 @@ arguments:
 
     atticmatic
 
-This will also prune any old backups as per the configured retention policy.
+This will also prune any old backups as per the configured retention policy,
+and check backups for consistency problems due to things like file damage.
+
 By default, the backup will proceed silently except in the case of errors. But
 if you'd like to to get additional information about the progress of the
 backup as it proceeds, use the verbose option instead:
diff --git a/atticmatic/attic.py b/atticmatic/attic.py
index c8842103c..d0c678d68 100644
--- a/atticmatic/attic.py
+++ b/atticmatic/attic.py
@@ -1,5 +1,5 @@
 from datetime import datetime
-
+import os
 import platform
 import subprocess
 
@@ -63,3 +63,19 @@ def prune_archives(verbose, repository, retention_config):
     ) + (('--verbose',) if verbose else ())
 
     subprocess.check_call(command)
+
+
+def check_archives(verbose, repository):
+    '''
+    Given a verbosity flag and a local or remote repository path, check the contained attic archives
+    for consistency.
+    '''
+    command = (
+        'attic', 'check',
+        repository,
+    ) + (('--verbose',) if verbose else ())
+
+    # Attic's check command spews to stdout even without the verbose flag. Suppress it.
+    stdout = None if verbose else open(os.devnull, 'w')
+
+    subprocess.check_call(command, stdout=stdout)
diff --git a/atticmatic/command.py b/atticmatic/command.py
index ff31ca786..92976f0ea 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -3,7 +3,7 @@ from argparse import ArgumentParser
 from subprocess import CalledProcessError
 import sys
 
-from atticmatic.attic import create_archive, prune_archives
+from atticmatic.attic import check_archives, create_archive, prune_archives
 from atticmatic.config import parse_configuration
 
 
@@ -41,9 +41,11 @@ def main():
     try:
         args = parse_arguments(*sys.argv[1:])
         location_config, retention_config = parse_configuration(args.config_filename)
+        repository = location_config['repository']
 
         create_archive(args.excludes_filename, args.verbose, **location_config)
-        prune_archives(args.verbose, location_config['repository'], retention_config)
+        prune_archives(args.verbose, repository, retention_config)
+        check_archives(args.verbose, repository)
     except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py
index 44b38bbd0..2c93e8c78 100644
--- a/atticmatic/tests/unit/test_attic.py
+++ b/atticmatic/tests/unit/test_attic.py
@@ -5,9 +5,9 @@ from flexmock import flexmock
 from atticmatic import attic as module
 
 
-def insert_subprocess_mock(check_call_command):
+def insert_subprocess_mock(check_call_command, **kwargs):
     subprocess = flexmock()
-    subprocess.should_receive('check_call').with_args(check_call_command).once()
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
     flexmock(module).subprocess = subprocess
 
 
@@ -111,3 +111,34 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
         verbose=True,
         retention_config=retention_config,
     )
+
+
+def test_check_archives_should_call_attic_with_parameters():
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('attic', 'check', 'repo'),
+        stdout=stdout,
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).open = lambda filename, mode: stdout
+    flexmock(module).os = flexmock().should_receive('devnull').mock
+
+    module.check_archives(
+        verbose=False,
+        repository='repo',
+    )
+
+
+def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
+    insert_subprocess_mock(
+        ('attic', 'check', 'repo', '--verbose'),
+        stdout=None,
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.check_archives(
+        verbose=True,
+        repository='repo',
+    )

From 9f5dd6c10deebff35c7eddaed1295a7ec5a71a0d Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 14 Feb 2015 09:24:15 -0800
Subject: [PATCH 030/189] Added tag 0.0.3 for changeset 7730ae34665c

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index ec58244b8..3f99142e7 100644
--- a/.hgtags
+++ b/.hgtags
@@ -1 +1,2 @@
 467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
+7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3

From f23810f19aa2d883453d390476d762892c631663 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 14 Feb 2015 09:31:42 -0800
Subject: [PATCH 031/189] Updating install instructions so you can upgrade from
 one release of atticmatic to the next.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4a2d2110c..99b8a5619 100644
--- a/README.md
+++ b/README.md
@@ -51,7 +51,7 @@ key-based ssh access to the desired user account on the remote host.
 
 To install atticmatic, run the following command to download and install it:
 
-    sudo pip install hg+https://torsion.org/hg/atticmatic
+    sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
 
 Then copy the following configuration files:
 

From 02df59e964675ccabe9a30f375a2f6dd886ee9b0 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 28 Feb 2015 11:03:22 -0800
Subject: [PATCH 032/189] Added a troubleshooting section with steps to deal
 with broken pipes.

---
 NEWS      |  4 ++++
 README.md | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/NEWS b/NEWS
index 93d33a5d6..0d2cb0eef 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.0.4-dev
+
+ * Added a troubleshooting section with steps to deal with broken pipes.
+
 0.0.3
 
  * After pruning, run attic's consistency checks on all archives.
diff --git a/README.md b/README.md
index 99b8a5619..d289369a6 100644
--- a/README.md
+++ b/README.md
@@ -94,6 +94,27 @@ To actually run tests, run:
     nosetests --detailed-errors
 
 
+## Troubleshooting
+
+### Broken pipe with remote repository
+
+When running atticmatic on a large remote repository, you may receive errors
+like the following, particularly while "attic check" is valiating backups for
+consistency:
+
+    Write failed: Broken pipe
+    attic: Error: Connection closed by remote host
+
+This error can be caused by an ssh timeout, which you can rectify by adding
+the following to the ~/.ssh/config file on the client:
+
+    Host *
+        ServerAliveInterval 120
+
+This should make the client keep the connection alive while validating
+backups.
+
+
 ## Feedback
 
 Questions? Comments? Got a patch? Contact <mailto:witten@torsion.org>.

From 2639b7105a28307a0a34c1728e1cecff1755c245 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 09:41:58 -0700
Subject: [PATCH 033/189] Added nosetests config file (setup.cfg) with
 defaults.

---
 NEWS      | 1 +
 README.md | 2 +-
 setup.cfg | 2 ++
 3 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 setup.cfg

diff --git a/NEWS b/NEWS
index 0d2cb0eef..c2a9dffce 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,7 @@
 0.0.4-dev
 
  * Added a troubleshooting section with steps to deal with broken pipes.
+ * Added nosetests config file (setup.cfg) with defaults.
 
 0.0.3
 
diff --git a/README.md b/README.md
index d289369a6..825ca5ee2 100644
--- a/README.md
+++ b/README.md
@@ -91,7 +91,7 @@ To install test-specific dependencies, first run:
 
 To actually run tests, run:
 
-    nosetests --detailed-errors
+    nosetests
 
 
 ## Troubleshooting
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 000000000..ead29d1ca
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,2 @@
+[nosetests]
+detailed-errors=1

From aa48b95ee7dadac145e9bf513b38e440f9a7bb68 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 09:52:40 -0700
Subject: [PATCH 034/189] Bumping setup.py version.

---
 NEWS                                |  1 +
 README.md                           | 31 ++++++-----
 atticmatic/attic.py                 | 12 ++++-
 atticmatic/command.py               |  2 +-
 atticmatic/tests/unit/test_attic.py | 81 +++++++++++++++++++++++++----
 setup.py                            |  2 +-
 6 files changed, 104 insertions(+), 25 deletions(-)

diff --git a/NEWS b/NEWS
index c2a9dffce..1464d2bdd 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
 0.0.4-dev
 
+ * Helpful error message about how to create a repository if one is missing.
  * Added a troubleshooting section with steps to deal with broken pipes.
  * Added nosetests config file (setup.cfg) with defaults.
 
diff --git a/README.md b/README.md
index 825ca5ee2..cd4298d5c 100644
--- a/README.md
+++ b/README.md
@@ -37,18 +37,6 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on
 
 ## Setup
 
-To get up and running with Attic, follow the [Attic Quick
-Start](https://attic-backup.org/quickstart.html) guide to create an Attic
-repository on a local or remote host. Note that if you plan to run atticmatic
-on a schedule with cron, and you encrypt your attic repository with a
-passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE`
-environment variable. See [attic's repository encryption
-documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
-more info.
-
-If the repository is on a remote host, make sure that your local root user has
-key-based ssh access to the desired user account on the remote host.
-
 To install atticmatic, run the following command to download and install it:
 
     sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
@@ -59,7 +47,24 @@ Then copy the following configuration files:
     sudo mkdir /etc/atticmatic/
     sudo cp sample/config sample/excludes /etc/atticmatic/
 
-Lastly, modify those files with your desired configuration.
+Modify those files with your desired configuration, including the path to an
+attic repository.
+
+If you don't yet have an attic repository, then the first time you run
+atticmatic, you'll get an error with information on how to create a repository
+on a local or remote host.
+
+And if the repository is on a remote host, make sure that your local root user
+has key-based ssh access to the desired user account on the remote host.
+
+It is recommended that you create your attic repository with keyfile
+encryption, as passphrase-based encryption is less suited for automated
+backups. If you do plan to run atticmatic on a schedule with cron, and you
+encrypt your attic repository with a passphrase instead of a key file, you'll
+need to set the `ATTIC_PASSPHRASE` environment variable. See [attic's
+repository encryption
+documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
+more info.
 
 
 ## Usage
diff --git a/atticmatic/attic.py b/atticmatic/attic.py
index d0c678d68..f0c9e7e4e 100644
--- a/atticmatic/attic.py
+++ b/atticmatic/attic.py
@@ -1,7 +1,10 @@
+from __future__ import print_function
 from datetime import datetime
 import os
 import platform
+import re
 import subprocess
+import sys
 
 
 def create_archive(excludes_filename, verbose, source_directories, repository):
@@ -23,7 +26,14 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
         ('--verbose', '--stats') if verbose else ()
     )
 
-    subprocess.check_call(command)
+    try:
+        subprocess.check_output(command, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError, error:
+        print(error.output.strip(), file=sys.stderr)
+
+        if re.search('Error: Repository .* does not exist', error.output):
+            raise RuntimeError('To create a repository, run: attic init --encryption=keyfile {}'.format(repository))
+        raise error
 
 
 def make_prune_flags(retention_config):
diff --git a/atticmatic/command.py b/atticmatic/command.py
index 92976f0ea..e0f4eb18b 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -46,6 +46,6 @@ def main():
         create_archive(args.excludes_filename, args.verbose, **location_config)
         prune_archives(args.verbose, repository, retention_config)
         check_archives(args.verbose, repository)
-    except (ValueError, IOError, CalledProcessError) as error:
+    except (ValueError, IOError, CalledProcessError, RuntimeError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py
index 2c93e8c78..30431cc00 100644
--- a/atticmatic/tests/unit/test_attic.py
+++ b/atticmatic/tests/unit/test_attic.py
@@ -1,14 +1,44 @@
 from collections import OrderedDict
+import sys
 
 from flexmock import flexmock
+from nose.tools import assert_raises
 
 from atticmatic import attic as module
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock()
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+class MockCalledProcessError(Exception):
+    def __init__(self, output):
+        self.output = output
+
+
+def insert_subprocess_check_output_mock(call_command, error_output=None, **kwargs):
+    subprocess = flexmock(CalledProcessError=MockCalledProcessError, STDOUT=flexmock())
+
+    expectation = subprocess.should_receive('check_output').with_args(
+        call_command,
+        stderr=subprocess.STDOUT,
+        **kwargs
+    ).once()
+
+    if error_output:
+        expectation.and_raise(MockCalledProcessError, output=error_output)
+        flexmock(sys.modules['__builtin__']).should_receive('print')
+
     flexmock(module).subprocess = subprocess
+    return subprocess
+
+
+def insert_subprocess_check_call_mock(call_command, **kwargs):
+    subprocess = flexmock()
+
+    subprocess.should_receive('check_call').with_args(
+        call_command,
+        **kwargs
+    ).once()
+
+    flexmock(module).subprocess = subprocess
+    return subprocess
 
 
 def insert_platform_mock():
@@ -22,7 +52,7 @@ def insert_datetime_mock():
 
 
 def test_create_archive_should_call_attic_with_parameters():
-    insert_subprocess_mock(
+    insert_subprocess_check_output_mock(
         ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
     )
     insert_platform_mock()
@@ -37,7 +67,7 @@ def test_create_archive_should_call_attic_with_parameters():
 
 
 def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters():
-    insert_subprocess_mock(
+    insert_subprocess_check_output_mock(
         (
             'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar',
             '--verbose', '--stats',
@@ -53,6 +83,39 @@ def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters()
         repository='repo',
     )
 
+def test_create_archive_with_missing_repository_should_raise():
+    insert_subprocess_check_output_mock(
+        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
+        error_output='Error: Repository repo does not exist',
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    with assert_raises(RuntimeError):
+        module.create_archive(
+            excludes_filename='excludes',
+            verbose=False,
+            source_directories='foo bar',
+            repository='repo',
+        )
+
+
+def test_create_archive_with_other_error_should_raise():
+    subprocess = insert_subprocess_check_output_mock(
+        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
+        error_output='Something went wrong',
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    with assert_raises(subprocess.CalledProcessError):
+        module.create_archive(
+            excludes_filename='excludes',
+            verbose=False,
+            source_directories='foo bar',
+            repository='repo',
+        )
+
 
 BASE_PRUNE_FLAGS = (
     ('--keep-daily', '1'),
@@ -80,7 +143,7 @@ def test_prune_archives_should_call_attic_with_parameters():
     flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_mock(
+    insert_subprocess_check_call_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
             '3',
@@ -99,7 +162,7 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
     flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_mock(
+    insert_subprocess_check_call_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
             '3', '--verbose',
@@ -115,7 +178,7 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
 
 def test_check_archives_should_call_attic_with_parameters():
     stdout = flexmock()
-    insert_subprocess_mock(
+    insert_subprocess_check_call_mock(
         ('attic', 'check', 'repo'),
         stdout=stdout,
     )
@@ -131,7 +194,7 @@ def test_check_archives_should_call_attic_with_parameters():
 
 
 def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
-    insert_subprocess_mock(
+    insert_subprocess_check_call_mock(
         ('attic', 'check', 'repo', '--verbose'),
         stdout=None,
     )
diff --git a/setup.py b/setup.py
index e73711233..9bcd56d21 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.2',
+    version='0.0.4',
     description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

From ee5697ac374619f8d640215cb07fa789b0f466f3 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:14:16 -0700
Subject: [PATCH 035/189] Fixing Python 3 test incompatibility with builtins.

---
 atticmatic/attic.py                 | 2 +-
 atticmatic/tests/unit/test_attic.py | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/atticmatic/attic.py b/atticmatic/attic.py
index f0c9e7e4e..d0b1cc896 100644
--- a/atticmatic/attic.py
+++ b/atticmatic/attic.py
@@ -28,7 +28,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
 
     try:
         subprocess.check_output(command, stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError, error:
+    except subprocess.CalledProcessError as error:
         print(error.output.strip(), file=sys.stderr)
 
         if re.search('Error: Repository .* does not exist', error.output):
diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py
index 30431cc00..e54f44573 100644
--- a/atticmatic/tests/unit/test_attic.py
+++ b/atticmatic/tests/unit/test_attic.py
@@ -1,5 +1,10 @@
 from collections import OrderedDict
-import sys
+try:
+    # Python 2
+    import __builtin__ as builtins
+except ImportError:
+    # Python 3
+    import builtins
 
 from flexmock import flexmock
 from nose.tools import assert_raises
@@ -23,7 +28,7 @@ def insert_subprocess_check_output_mock(call_command, error_output=None, **kwarg
 
     if error_output:
         expectation.and_raise(MockCalledProcessError, output=error_output)
-        flexmock(sys.modules['__builtin__']).should_receive('print')
+        flexmock(builtins).should_receive('print')
 
     flexmock(module).subprocess = subprocess
     return subprocess

From 715b240589c76ff03cdedc9e74899a95e92b1352 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:14:30 -0700
Subject: [PATCH 036/189] Now using tox to run tests against multiple versions
 of Python in one go.

---
 .hgignore             | 1 +
 NEWS                  | 5 +++--
 README.md             | 8 ++++----
 test_requirements.txt | 2 ++
 tox.ini               | 8 ++++++++
 5 files changed, 18 insertions(+), 6 deletions(-)
 create mode 100644 test_requirements.txt
 create mode 100644 tox.ini

diff --git a/.hgignore b/.hgignore
index abab07771..d7c266a54 100644
--- a/.hgignore
+++ b/.hgignore
@@ -2,3 +2,4 @@ syntax: glob
 *.egg-info
 *.pyc
 *.swp
+.tox
diff --git a/NEWS b/NEWS
index 1464d2bdd..624ad2754 100644
--- a/NEWS
+++ b/NEWS
@@ -1,8 +1,9 @@
 0.0.4-dev
 
+ * Now using tox to run tests against multiple versions of Python in one go.
  * Helpful error message about how to create a repository if one is missing.
- * Added a troubleshooting section with steps to deal with broken pipes.
- * Added nosetests config file (setup.cfg) with defaults.
+ * Troubleshooting section with steps to deal with broken pipes.
+ * Nosetests config file (setup.cfg) with defaults.
 
 0.0.3
 
diff --git a/README.md b/README.md
index cd4298d5c..c60827302 100644
--- a/README.md
+++ b/README.md
@@ -90,13 +90,13 @@ If you'd like to see the available command-line arguments, view the help:
 
 ## Running tests
 
-To install test-specific dependencies, first run:
+First install tox, which is used for setting up testing environments:
 
-    sudo python setup.py test
+    pip install tox
 
-To actually run tests, run:
+Then, to actually run tests, run:
 
-    nosetests
+    tox
 
 
 ## Troubleshooting
diff --git a/test_requirements.txt b/test_requirements.txt
new file mode 100644
index 000000000..5a34a857b
--- /dev/null
+++ b/test_requirements.txt
@@ -0,0 +1,2 @@
+flexmock==0.9.7
+nose==1.3.4
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 000000000..daf5e8d05
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,8 @@
+[tox]
+envlist=py27,py34
+skipsdist=True
+
+[testenv]
+usedevelop=True
+deps=-rtest_requirements.txt
+commands = nosetests

From 66286f92df9275e8e614d7aac1f8b192041d5cfb Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:15:03 -0700
Subject: [PATCH 037/189] Releasing 0.0.4.

---
 NEWS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 624ad2754..e5ba5345b 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
-0.0.4-dev
+0.0.4
 
  * Now using tox to run tests against multiple versions of Python in one go.
  * Helpful error message about how to create a repository if one is missing.

From d9e396e26415a1df5047ea40dcf9e9e6a383b532 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:19:12 -0700
Subject: [PATCH 038/189] Added tag 0.0.4 for changeset 4bb2e81fc770

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 3f99142e7..d101a2abc 100644
--- a/.hgtags
+++ b/.hgtags
@@ -1,2 +1,3 @@
 467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
 7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3
+4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4

From bda6451c1d6be0ff105f83d32aca4b4ba516a0cb Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:39:08 -0700
Subject: [PATCH 039/189] Added tag 0.0.5 for changeset b31d51b63370

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index d101a2abc..a4c57269e 100644
--- a/.hgtags
+++ b/.hgtags
@@ -1,3 +1,4 @@
 467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
 7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3
 4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4
+b31d51b633701554e84f996cc0c73bad2990780b 0.0.5

From ac6c927a23865411da85f489e60afffc4c0db7da Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:44:18 -0700
Subject: [PATCH 040/189] Backout out "helpful" error message that broke
 --verbose.

---
 NEWS                                |  5 ++
 README.md                           | 31 +++++------
 atticmatic/attic.py                 | 12 +----
 atticmatic/command.py               |  2 +-
 atticmatic/tests/unit/test_attic.py | 84 +++--------------------------
 setup.py                            |  2 +-
 6 files changed, 29 insertions(+), 107 deletions(-)

diff --git a/NEWS b/NEWS
index e5ba5345b..43351a5ce 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+0.0.5
+
+ * Fixed regression with --verbose output being buffered. This means dropping the helpful error
+   message introduced in 0.0.4.
+
 0.0.4
 
  * Now using tox to run tests against multiple versions of Python in one go.
diff --git a/README.md b/README.md
index c60827302..f39964623 100644
--- a/README.md
+++ b/README.md
@@ -37,6 +37,18 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on
 
 ## Setup
 
+To get up and running with Attic, follow the [Attic Quick
+Start](https://attic-backup.org/quickstart.html) guide to create an Attic
+repository on a local or remote host. Note that if you plan to run atticmatic
+on a schedule with cron, and you encrypt your attic repository with a
+passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE`
+environment variable. See [attic's repository encryption
+documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
+more info.
+
+If the repository is on a remote host, make sure that your local root user has
+key-based ssh access to the desired user account on the remote host.
+
 To install atticmatic, run the following command to download and install it:
 
     sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
@@ -47,24 +59,7 @@ Then copy the following configuration files:
     sudo mkdir /etc/atticmatic/
     sudo cp sample/config sample/excludes /etc/atticmatic/
 
-Modify those files with your desired configuration, including the path to an
-attic repository.
-
-If you don't yet have an attic repository, then the first time you run
-atticmatic, you'll get an error with information on how to create a repository
-on a local or remote host.
-
-And if the repository is on a remote host, make sure that your local root user
-has key-based ssh access to the desired user account on the remote host.
-
-It is recommended that you create your attic repository with keyfile
-encryption, as passphrase-based encryption is less suited for automated
-backups. If you do plan to run atticmatic on a schedule with cron, and you
-encrypt your attic repository with a passphrase instead of a key file, you'll
-need to set the `ATTIC_PASSPHRASE` environment variable. See [attic's
-repository encryption
-documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
-more info.
+Lastly, modify those files with your desired configuration.
 
 
 ## Usage
diff --git a/atticmatic/attic.py b/atticmatic/attic.py
index d0b1cc896..d0c678d68 100644
--- a/atticmatic/attic.py
+++ b/atticmatic/attic.py
@@ -1,10 +1,7 @@
-from __future__ import print_function
 from datetime import datetime
 import os
 import platform
-import re
 import subprocess
-import sys
 
 
 def create_archive(excludes_filename, verbose, source_directories, repository):
@@ -26,14 +23,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
         ('--verbose', '--stats') if verbose else ()
     )
 
-    try:
-        subprocess.check_output(command, stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as error:
-        print(error.output.strip(), file=sys.stderr)
-
-        if re.search('Error: Repository .* does not exist', error.output):
-            raise RuntimeError('To create a repository, run: attic init --encryption=keyfile {}'.format(repository))
-        raise error
+    subprocess.check_call(command)
 
 
 def make_prune_flags(retention_config):
diff --git a/atticmatic/command.py b/atticmatic/command.py
index e0f4eb18b..92976f0ea 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -46,6 +46,6 @@ def main():
         create_archive(args.excludes_filename, args.verbose, **location_config)
         prune_archives(args.verbose, repository, retention_config)
         check_archives(args.verbose, repository)
-    except (ValueError, IOError, CalledProcessError, RuntimeError) as error:
+    except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py
index e54f44573..2c93e8c78 100644
--- a/atticmatic/tests/unit/test_attic.py
+++ b/atticmatic/tests/unit/test_attic.py
@@ -1,49 +1,14 @@
 from collections import OrderedDict
-try:
-    # Python 2
-    import __builtin__ as builtins
-except ImportError:
-    # Python 3
-    import builtins
 
 from flexmock import flexmock
-from nose.tools import assert_raises
 
 from atticmatic import attic as module
 
 
-class MockCalledProcessError(Exception):
-    def __init__(self, output):
-        self.output = output
-
-
-def insert_subprocess_check_output_mock(call_command, error_output=None, **kwargs):
-    subprocess = flexmock(CalledProcessError=MockCalledProcessError, STDOUT=flexmock())
-
-    expectation = subprocess.should_receive('check_output').with_args(
-        call_command,
-        stderr=subprocess.STDOUT,
-        **kwargs
-    ).once()
-
-    if error_output:
-        expectation.and_raise(MockCalledProcessError, output=error_output)
-        flexmock(builtins).should_receive('print')
-
-    flexmock(module).subprocess = subprocess
-    return subprocess
-
-
-def insert_subprocess_check_call_mock(call_command, **kwargs):
+def insert_subprocess_mock(check_call_command, **kwargs):
     subprocess = flexmock()
-
-    subprocess.should_receive('check_call').with_args(
-        call_command,
-        **kwargs
-    ).once()
-
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
     flexmock(module).subprocess = subprocess
-    return subprocess
 
 
 def insert_platform_mock():
@@ -57,7 +22,7 @@ def insert_datetime_mock():
 
 
 def test_create_archive_should_call_attic_with_parameters():
-    insert_subprocess_check_output_mock(
+    insert_subprocess_mock(
         ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
     )
     insert_platform_mock()
@@ -72,7 +37,7 @@ def test_create_archive_should_call_attic_with_parameters():
 
 
 def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters():
-    insert_subprocess_check_output_mock(
+    insert_subprocess_mock(
         (
             'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar',
             '--verbose', '--stats',
@@ -88,39 +53,6 @@ def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters()
         repository='repo',
     )
 
-def test_create_archive_with_missing_repository_should_raise():
-    insert_subprocess_check_output_mock(
-        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
-        error_output='Error: Repository repo does not exist',
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    with assert_raises(RuntimeError):
-        module.create_archive(
-            excludes_filename='excludes',
-            verbose=False,
-            source_directories='foo bar',
-            repository='repo',
-        )
-
-
-def test_create_archive_with_other_error_should_raise():
-    subprocess = insert_subprocess_check_output_mock(
-        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
-        error_output='Something went wrong',
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    with assert_raises(subprocess.CalledProcessError):
-        module.create_archive(
-            excludes_filename='excludes',
-            verbose=False,
-            source_directories='foo bar',
-            repository='repo',
-        )
-
 
 BASE_PRUNE_FLAGS = (
     ('--keep-daily', '1'),
@@ -148,7 +80,7 @@ def test_prune_archives_should_call_attic_with_parameters():
     flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_check_call_mock(
+    insert_subprocess_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
             '3',
@@ -167,7 +99,7 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
     flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_check_call_mock(
+    insert_subprocess_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
             '3', '--verbose',
@@ -183,7 +115,7 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
 
 def test_check_archives_should_call_attic_with_parameters():
     stdout = flexmock()
-    insert_subprocess_check_call_mock(
+    insert_subprocess_mock(
         ('attic', 'check', 'repo'),
         stdout=stdout,
     )
@@ -199,7 +131,7 @@ def test_check_archives_should_call_attic_with_parameters():
 
 
 def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
-    insert_subprocess_check_call_mock(
+    insert_subprocess_mock(
         ('attic', 'check', 'repo', '--verbose'),
         stdout=None,
     )
diff --git a/setup.py b/setup.py
index 9bcd56d21..e73711233 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.4',
+    version='0.0.2',
     description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

From 3506819511f01612f781d1a5baa7c2fcdf931b98 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:44:25 -0700
Subject: [PATCH 041/189] Added tag 0.0.5 for changeset aa8a807f4ba2

---
 .hgtags | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.hgtags b/.hgtags
index a4c57269e..023f01fae 100644
--- a/.hgtags
+++ b/.hgtags
@@ -2,3 +2,5 @@
 7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3
 4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4
 b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
+b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
+aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5

From aa1178dc492382379ee20a772d3150e154506358 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:46:55 -0700
Subject: [PATCH 042/189] Added tag 0.0.5 for changeset 569aef47a9b2

---
 .hgtags | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.hgtags b/.hgtags
index 023f01fae..1d304c79d 100644
--- a/.hgtags
+++ b/.hgtags
@@ -4,3 +4,5 @@
 b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
 b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
 aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
+aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
+569aef47a9b25c55b13753f94706f5d330219995 0.0.5

From cb402d68464d78da6d47ba3bfa09f482d4482852 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:47:49 -0700
Subject: [PATCH 043/189] Re-fixing version.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index e73711233..a771454fb 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.2',
+    version='0.0.5',
     description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

From 4e4f8c2670dc4a25d0b8b2d84c18b9cdc54e9855 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 10:47:58 -0700
Subject: [PATCH 044/189] Added tag 0.0.5 for changeset a03495a8e8b4

---
 .hgtags | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.hgtags b/.hgtags
index 1d304c79d..f3659ea44 100644
--- a/.hgtags
+++ b/.hgtags
@@ -6,3 +6,5 @@ b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
 aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
 aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
 569aef47a9b25c55b13753f94706f5d330219995 0.0.5
+569aef47a9b25c55b13753f94706f5d330219995 0.0.5
+a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5

From 7750d2568c8d118d446181482518a9aba9317c2b Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 15 Mar 2015 11:15:40 -0700
Subject: [PATCH 045/189] Passing through command-line options from tox to
 nosetests.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index daf5e8d05..6ac577bbf 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,4 +5,4 @@ skipsdist=True
 [testenv]
 usedevelop=True
 deps=-rtest_requirements.txt
-commands = nosetests
+commands = nosetests []

From cfd61dc1d15cc41d71c775197d17c07d52394978 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sun, 10 May 2015 22:00:31 -0700
Subject: [PATCH 046/189] New configuration section for customizing which Attic
 consistency checks run, if any.

---
 NEWS                                 |  4 ++
 README.md                            |  3 ++
 atticmatic/attic.py                  | 67 +++++++++++++++++++++++---
 atticmatic/command.py                | 10 ++--
 atticmatic/config.py                 | 51 +++++++++++++++-----
 atticmatic/tests/unit/test_attic.py  | 70 ++++++++++++++++++++++++++--
 atticmatic/tests/unit/test_config.py | 70 +++++++++++++++++++++++++---
 sample/config                        | 10 +++-
 setup.py                             |  2 +-
 9 files changed, 251 insertions(+), 36 deletions(-)

diff --git a/NEWS b/NEWS
index 43351a5ce..5dbc3bcbc 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.0.6
+
+ * New configuration section for customizing which Attic consistency checks run, if any.
+
 0.0.5
 
  * Fixed regression with --verbose output being buffered. This means dropping the helpful error
diff --git a/README.md b/README.md
index f39964623..ee9cb722a 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,9 @@ Here's an example config file:
     keep_weekly: 4
     keep_monthly: 6
 
+    [consistency]
+    checks: repository archives
+
 Additionally, exclude patterns can be specified in a separate excludes config
 file, one pattern per line.
 
diff --git a/atticmatic/attic.py b/atticmatic/attic.py
index d0c678d68..fc8399781 100644
--- a/atticmatic/attic.py
+++ b/atticmatic/attic.py
@@ -26,7 +26,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
     subprocess.check_call(command)
 
 
-def make_prune_flags(retention_config):
+def _make_prune_flags(retention_config):
     '''
     Given a retention config dict mapping from option name to value, tranform it into an iterable of
     command-line name-value flag pairs.
@@ -58,22 +58,77 @@ def prune_archives(verbose, repository, retention_config):
         repository,
     ) + tuple(
         element
-        for pair in make_prune_flags(retention_config)
+        for pair in _make_prune_flags(retention_config)
         for element in pair
     ) + (('--verbose',) if verbose else ())
 
     subprocess.check_call(command)
 
 
-def check_archives(verbose, repository):
+DEFAULT_CHECKS = ('repository', 'archives')
+
+
+def _parse_checks(consistency_config):
     '''
-    Given a verbosity flag and a local or remote repository path, check the contained attic archives
-    for consistency.
+    Given a consistency config with a space-separated "checks" option, transform it to a tuple of
+    named checks to run.
+
+    For example, given a retention config of:
+
+        {'checks': 'repository archives'}
+
+    This will be returned as:
+
+        ('repository', 'archives')
+
+    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
+    "disabled", return an empty tuple, meaning that no checks should be run.
     '''
+    checks = consistency_config.get('checks', '').strip()
+    if not checks:
+        return DEFAULT_CHECKS
+
+    return tuple(
+        check for check in consistency_config['checks'].split(' ')
+        if check.lower() not in ('disabled', '')
+    )
+
+
+def _make_check_flags(checks):
+    '''
+    Given a parsed sequence of checks, transform it into tuple of command-line flags.
+
+    For example, given parsed checks of:
+
+        ('repository',)
+
+    This will be returned as:
+    
+        ('--repository-only',)
+    '''
+    if checks == DEFAULT_CHECKS:
+        return ()
+
+    return tuple(
+        '--{}-only'.format(check) for check in checks
+    )
+
+
+def check_archives(verbose, repository, consistency_config):
+    '''
+    Given a verbosity flag, a local or remote repository path, and a consistency config dict, check
+    the contained attic archives for consistency.
+
+    If there are no consistency checks to run, skip running them.
+    '''
+    checks = _parse_checks(consistency_config)
+    if not checks:
+        return
+
     command = (
         'attic', 'check',
         repository,
-    ) + (('--verbose',) if verbose else ())
+    ) + _make_check_flags(checks) + (('--verbose',) if verbose else ())
 
     # Attic's check command spews to stdout even without the verbose flag. Suppress it.
     stdout = None if verbose else open(os.devnull, 'w')
diff --git a/atticmatic/command.py b/atticmatic/command.py
index 92976f0ea..0a844b20f 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -40,12 +40,12 @@ def parse_arguments(*arguments):
 def main():
     try:
         args = parse_arguments(*sys.argv[1:])
-        location_config, retention_config = parse_configuration(args.config_filename)
-        repository = location_config['repository']
+        config = parse_configuration(args.config_filename)
+        repository = config.location['repository']
 
-        create_archive(args.excludes_filename, args.verbose, **location_config)
-        prune_archives(args.verbose, repository, retention_config)
-        check_archives(args.verbose, repository)
+        create_archive(args.excludes_filename, args.verbose, **config.location)
+        prune_archives(args.verbose, repository, config.retention)
+        check_archives(args.verbose, repository, config.consistency)
     except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/atticmatic/config.py b/atticmatic/config.py
index f953860a9..8254c8804 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -39,6 +39,12 @@ CONFIG_FORMAT = (
             option('keep_yearly', int, required=False),
             option('prefix', required=False),
         ),
+    ),
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+        ),
     )
 )
 
@@ -49,20 +55,34 @@ def validate_configuration_format(parser, config_format):
     configuration file has the expected sections, that any required options are present in those
     sections, and that there aren't any unexpected options.
 
+    A section is required if any of its contained options are required.
+
     Raise ValueError if anything is awry.
     '''
-    section_names = parser.sections()
-    required_section_names = tuple(section.name for section in config_format)
+    section_names = set(parser.sections())
+    required_section_names = tuple(
+        section.name for section in config_format
+        if any(option.required for option in section.options)
+    )
 
-    if set(section_names) != set(required_section_names):
+    unknown_section_names = section_names - set(
+        section_format.name for section_format in config_format
+    )
+    if unknown_section_names:
         raise ValueError(
-            'Expected config sections {} but found sections: {}'.format(
-                ', '.join(required_section_names),
-                ', '.join(section_names)
-            )
+            'Unknown config sections found: {}'.format(', '.join(unknown_section_names))
+        )
+
+    missing_section_names = set(required_section_names) - section_names
+    if missing_section_names:
+        raise ValueError(
+            'Missing config sections: {}'.format(', '.join(missing_section_names))
         )
 
     for section_format in config_format:
+        if section_format.name not in section_names:
+            continue
+
         option_names = parser.options(section_format.name)
         expected_options = section_format.options
 
@@ -90,6 +110,11 @@ def validate_configuration_format(parser, config_format):
             )
 
 
+# Describes a parsed configuration, where each attribute is the name of a configuration file section 
+# and each value is a dict of that section's parsed options.
+Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
+
+
 def parse_section_options(parser, section_format):
     '''
     Given an open ConfigParser and an expected section format, return the option values from that
@@ -112,8 +137,8 @@ def parse_section_options(parser, section_format):
 
 def parse_configuration(config_filename):
     '''
-    Given a config filename of the expected format, return the parsed configuration as a tuple of
-    (location config, retention config) where each config is a dict of that section's options.
+    Given a config filename of the expected format, return the parsed configuration as Parsed_config
+    data structure.
 
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
@@ -122,7 +147,9 @@ def parse_configuration(config_filename):
 
     validate_configuration_format(parser, CONFIG_FORMAT)
 
-    return tuple(
-        parse_section_options(parser, section_format)
-        for section_format in CONFIG_FORMAT
+    return Parsed_config(
+        *(
+            parse_section_options(parser, section_format)
+            for section_format in CONFIG_FORMAT
+        )
     )
diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py
index 2c93e8c78..26c629984 100644
--- a/atticmatic/tests/unit/test_attic.py
+++ b/atticmatic/tests/unit/test_attic.py
@@ -11,6 +11,12 @@ def insert_subprocess_mock(check_call_command, **kwargs):
     flexmock(module).subprocess = subprocess
 
 
+def insert_subprocess_never():
+    subprocess = flexmock()
+    subprocess.should_receive('check_call').never()
+    flexmock(module).subprocess = subprocess
+
+
 def insert_platform_mock():
     flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock
 
@@ -70,14 +76,14 @@ def test_make_prune_flags_should_return_flags_from_config():
         )
     )
 
-    result = module.make_prune_flags(retention_config)
+    result = module._make_prune_flags(retention_config)
 
     assert tuple(result) == BASE_PRUNE_FLAGS
 
 
 def test_prune_archives_should_call_attic_with_parameters():
     retention_config = flexmock()
-    flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
     insert_subprocess_mock(
@@ -96,7 +102,7 @@ def test_prune_archives_should_call_attic_with_parameters():
 
 def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters():
     retention_config = flexmock()
-    flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
     insert_subprocess_mock(
@@ -113,7 +119,46 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
     )
 
 
+def test_parse_checks_returns_them_as_tuple():
+    checks = module._parse_checks({'checks': 'foo disabled bar'})
+
+    assert checks == ('foo', 'bar')
+
+
+def test_parse_checks_with_missing_value_returns_defaults():
+    checks = module._parse_checks({})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_blank_value_returns_defaults():
+    checks = module._parse_checks({'checks': ''})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_disabled_returns_no_checks():
+    checks = module._parse_checks({'checks': 'disabled'})
+
+    assert checks == ()
+
+
+def test_make_check_flags_with_checks_returns_flags():
+    flags = module._make_check_flags(('foo', 'bar'))
+
+    assert flags == ('--foo-only', '--bar-only')
+
+
+def test_make_check_flags_with_default_checks_returns_no_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS)
+
+    assert flags == ()
+
+
 def test_check_archives_should_call_attic_with_parameters():
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_make_check_flags').and_return(())
     stdout = flexmock()
     insert_subprocess_mock(
         ('attic', 'check', 'repo'),
@@ -127,10 +172,14 @@ def test_check_archives_should_call_attic_with_parameters():
     module.check_archives(
         verbose=False,
         repository='repo',
+        consistency_config=consistency_config,
     )
 
 
 def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
         ('attic', 'check', 'repo', '--verbose'),
         stdout=None,
@@ -141,4 +190,19 @@ def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters()
     module.check_archives(
         verbose=True,
         repository='repo',
+        consistency_config=consistency_config,
     )
+
+
+def test_check_archives_without_any_checks_should_bail():
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(())
+    insert_subprocess_never()
+
+    module.check_archives(
+        verbose=False,
+        repository='repo',
+        consistency_config=consistency_config,
+    )
+
+
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index 0576dc8f6..1c9fcb654 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -41,19 +41,55 @@ def test_validate_configuration_format_with_valid_config_should_not_raise():
     module.validate_configuration_format(parser, config_format)
 
 
-def test_validate_configuration_format_with_missing_section_should_raise():
+def test_validate_configuration_format_with_missing_required_section_should_raise():
     parser = flexmock()
     parser.should_receive('sections').and_return(('section',))
     config_format = (
-        module.Section_format('section', options=()),
-        module.Section_format('missing', options=()),
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        # At least one option in this section is required, so the section is required.
+        module.Section_format(
+            'missing',
+            options=(
+                module.Config_option('such', str, required=False),
+                module.Config_option('things', str, required=True),
+            ),
+        ),
     )
 
     with assert_raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
-def test_validate_configuration_format_with_extra_section_should_raise():
+def test_validate_configuration_format_with_missing_optional_section_should_not_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('stuff',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        # No options in the section are required, so the section is optional.
+        module.Section_format(
+            'missing',
+            options=(
+                module.Config_option('such', str, required=False),
+                module.Config_option('things', str, required=False),
+            ),
+        ),
+    )
+
+    module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_unknown_section_should_raise():
     parser = flexmock()
     parser.should_receive('sections').and_return(('section', 'extra'))
     config_format = (
@@ -139,6 +175,26 @@ def test_parse_section_options_should_return_section_options():
     )
 
 
+def test_parse_section_options_for_missing_section_should_return_empty_dict():
+    parser = flexmock()
+    parser.should_receive('get').never()
+    parser.should_receive('getint').never()
+    parser.should_receive('has_option').with_args('section', 'foo').and_return(False)
+    parser.should_receive('has_option').with_args('section', 'bar').and_return(False)
+
+    section_format = module.Section_format(
+        'section',
+        (
+            module.Config_option('foo', str, required=False),
+            module.Config_option('bar', int, required=False),
+        ),
+    )
+
+    config = module.parse_section_options(parser, section_format)
+
+    assert config == OrderedDict()
+
+
 def insert_mock_parser():
     parser = flexmock()
     parser.should_receive('readfp')
@@ -154,13 +210,13 @@ def test_parse_configuration_should_return_section_configs():
     mock_module.should_receive('validate_configuration_format').with_args(
         parser, module.CONFIG_FORMAT,
     ).once()
-    mock_section_configs = (flexmock(), flexmock())
+    mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT)
 
     for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
         mock_module.should_receive('parse_section_options').with_args(
             parser, section_format,
         ).and_return(section_config).once()
 
-    section_configs = module.parse_configuration('filename')
+    parsed_config = module.parse_configuration('filename')
 
-    assert section_configs == mock_section_configs
+    assert parsed_config == module.Parsed_config(*mock_section_configs)
diff --git a/sample/config b/sample/config
index aa2acf3f5..b13afbe60 100644
--- a/sample/config
+++ b/sample/config
@@ -6,8 +6,8 @@ source_directories: /home /etc
 repository: user@backupserver:sourcehostname.attic
 
 [retention]
-# Retention policy for how many backups to keep in each category.
-# See https://attic-backup.org/usage.html#attic-prune for details.
+# Retention policy for how many backups to keep in each category. See
+# https://attic-backup.org/usage.html#attic-prune for details.
 #keep_within: 3h
 #keep_hourly: 24
 keep_daily: 7
@@ -15,3 +15,9 @@ keep_weekly: 4
 keep_monthly: 6
 keep_yearly: 1
 #prefix: sourcehostname
+
+[consistency]
+# Space-separated list of consistency checks to run: "repository", "archives",
+# or both. Defaults to both. Set to "disabled" to disable all consistency
+# checks. See https://attic-backup.org/usage.html#attic-check for details.
+checks: repository archives
diff --git a/setup.py b/setup.py
index a771454fb..6fcd51d7c 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.5',
+    version='0.0.6',
     description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

From c8f1af635f3660b4c6bfc7648437e8b455bbf812 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sun, 10 May 2015 22:00:34 -0700
Subject: [PATCH 047/189] Added tag 0.0.6 for changeset 7ea93ca83f42

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index f3659ea44..875f102b8 100644
--- a/.hgtags
+++ b/.hgtags
@@ -8,3 +8,4 @@ aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
 569aef47a9b25c55b13753f94706f5d330219995 0.0.5
 569aef47a9b25c55b13753f94706f5d330219995 0.0.5
 a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
+7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6

From c3613e0637fb9ecd4e41dd2b8d529956d8af0022 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sun, 10 May 2015 22:06:48 -0700
Subject: [PATCH 048/189] Adding some explanitory text about consistency checks
 to README example.

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index ee9cb722a..771ca53cc 100644
--- a/README.md
+++ b/README.md
@@ -27,6 +27,7 @@ Here's an example config file:
     keep_monthly: 6
 
     [consistency]
+    # Consistency checks to run, or "disabled" to prevent checks.
     checks: repository archives
 
 Additionally, exclude patterns can be specified in a separate excludes config

From d9125451f5987b1751aba582491f9063d1496f8d Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sun, 14 Jun 2015 11:00:46 -0700
Subject: [PATCH 049/189] Improved mocking of Python builtins in unit tests.

---
 NEWS                                 |  4 ++++
 atticmatic/config.py                 |  2 +-
 atticmatic/tests/builtins.py         | 11 +++++++++++
 atticmatic/tests/unit/test_attic.py  |  9 ++++-----
 atticmatic/tests/unit/test_config.py |  5 ++---
 5 files changed, 22 insertions(+), 9 deletions(-)
 create mode 100644 atticmatic/tests/builtins.py

diff --git a/NEWS b/NEWS
index 5dbc3bcbc..302227ff1 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.0.7-dev
+
+ * Improved mocking of Python builtins in unit tests.
+
 0.0.6
 
  * New configuration section for customizing which Attic consistency checks run, if any.
diff --git a/atticmatic/config.py b/atticmatic/config.py
index 8254c8804..7474f0502 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -143,7 +143,7 @@ def parse_configuration(config_filename):
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
     parser = ConfigParser()
-    parser.readfp(open(config_filename))
+    parser.read(config_filename)
 
     validate_configuration_format(parser, CONFIG_FORMAT)
 
diff --git a/atticmatic/tests/builtins.py b/atticmatic/tests/builtins.py
new file mode 100644
index 000000000..ff48f939b
--- /dev/null
+++ b/atticmatic/tests/builtins.py
@@ -0,0 +1,11 @@
+from flexmock import flexmock
+import sys
+
+
+def builtins_mock():
+    try:
+        # Python 2
+        return flexmock(sys.modules['__builtin__'])
+    except KeyError:
+        # Python 3
+        return flexmock(sys.modules['builtins'])
diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py
index 26c629984..ab52850c4 100644
--- a/atticmatic/tests/unit/test_attic.py
+++ b/atticmatic/tests/unit/test_attic.py
@@ -3,6 +3,7 @@ from collections import OrderedDict
 from flexmock import flexmock
 
 from atticmatic import attic as module
+from atticmatic.tests.builtins import builtins_mock
 
 
 def insert_subprocess_mock(check_call_command, **kwargs):
@@ -18,7 +19,7 @@ def insert_subprocess_never():
 
 
 def insert_platform_mock():
-    flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock
+    flexmock(module.platform).should_receive('node').and_return('host')
 
 
 def insert_datetime_mock():
@@ -166,8 +167,8 @@ def test_check_archives_should_call_attic_with_parameters():
     )
     insert_platform_mock()
     insert_datetime_mock()
-    flexmock(module).open = lambda filename, mode: stdout
-    flexmock(module).os = flexmock().should_receive('devnull').mock
+    builtins_mock().should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
 
     module.check_archives(
         verbose=False,
@@ -204,5 +205,3 @@ def test_check_archives_without_any_checks_should_bail():
         repository='repo',
         consistency_config=consistency_config,
     )
-
-
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index 1c9fcb654..02f1f521b 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -197,9 +197,8 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict():
 
 def insert_mock_parser():
     parser = flexmock()
-    parser.should_receive('readfp')
-    flexmock(module).open = lambda filename: None
-    flexmock(module).ConfigParser = parser
+    parser.should_receive('read')
+    module.ConfigParser = lambda: parser
 
     return parser
 

From 5bf3a4875c596574bf57a476108eb19c881b88ec Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Fri, 17 Jul 2015 21:58:50 -0700
Subject: [PATCH 050/189] Flag for multiple levels of verbosity: some, and
 lots.

---
 NEWS                                         |  3 +-
 README.md                                    |  8 +-
 atticmatic/attic.py                          | 32 +++++--
 atticmatic/command.py                        | 12 +--
 atticmatic/tests/integration/test_command.py | 10 +-
 atticmatic/tests/unit/test_attic.py          | 99 ++++++++++++++------
 atticmatic/verbosity.py                      |  2 +
 setup.py                                     |  2 +-
 8 files changed, 113 insertions(+), 55 deletions(-)
 create mode 100644 atticmatic/verbosity.py

diff --git a/NEWS b/NEWS
index 302227ff1..f46598a4f 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
-0.0.7-dev
+0.0.7
 
+ * Flag for multiple levels of verbosity: some, and lots.
  * Improved mocking of Python builtins in unit tests.
 
 0.0.6
diff --git a/README.md b/README.md
index 771ca53cc..67b4bc1e4 100644
--- a/README.md
+++ b/README.md
@@ -78,9 +78,13 @@ and check backups for consistency problems due to things like file damage.
 
 By default, the backup will proceed silently except in the case of errors. But
 if you'd like to to get additional information about the progress of the
-backup as it proceeds, use the verbose option instead:
+backup as it proceeds, use the verbosity option:
 
-    atticmattic --verbose
+    atticmattic --verbosity 1
+
+Or, for even more progress spew:
+
+    atticmattic --verbosity 2
 
 If you'd like to see the available command-line arguments, view the help:
 
diff --git a/atticmatic/attic.py b/atticmatic/attic.py
index fc8399781..5a0a4b772 100644
--- a/atticmatic/attic.py
+++ b/atticmatic/attic.py
@@ -3,13 +3,19 @@ import os
 import platform
 import subprocess
 
+from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
-def create_archive(excludes_filename, verbose, source_directories, repository):
+
+def create_archive(excludes_filename, verbosity, source_directories, repository):
     '''
     Given an excludes filename, a vebosity flag, a space-separated list of source directories, and
     a local or remote repository path, create an attic archive.
     '''
     sources = tuple(source_directories.split(' '))
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--stats',),
+        VERBOSITY_LOTS: ('--verbose', '--stats'),
+    }.get(verbosity, ())
 
     command = (
         'attic', 'create',
@@ -19,9 +25,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
             hostname=platform.node(),
             timestamp=datetime.now().isoformat(),
         ),
-    ) + sources + (
-        ('--verbose', '--stats') if verbose else ()
-    )
+    ) + sources + verbosity_flags
 
     subprocess.check_call(command)
 
@@ -48,11 +52,16 @@ def _make_prune_flags(retention_config):
     )
 
 
-def prune_archives(verbose, repository, retention_config):
+def prune_archives(verbosity, repository, retention_config):
     '''
     Given a verbosity flag, a local or remote repository path, and a retention config dict, prune
     attic archives according the the retention policy specified in that configuration.
     '''
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--stats',),
+        VERBOSITY_LOTS: ('--verbose', '--stats'),
+    }.get(verbosity, ())
+
     command = (
         'attic', 'prune',
         repository,
@@ -60,7 +69,7 @@ def prune_archives(verbose, repository, retention_config):
         element
         for pair in _make_prune_flags(retention_config)
         for element in pair
-    ) + (('--verbose',) if verbose else ())
+    ) + verbosity_flags
 
     subprocess.check_call(command)
 
@@ -114,7 +123,7 @@ def _make_check_flags(checks):
     )
 
 
-def check_archives(verbose, repository, consistency_config):
+def check_archives(verbosity, repository, consistency_config):
     '''
     Given a verbosity flag, a local or remote repository path, and a consistency config dict, check
     the contained attic archives for consistency.
@@ -125,12 +134,17 @@ def check_archives(verbose, repository, consistency_config):
     if not checks:
         return
 
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--verbose',),
+        VERBOSITY_LOTS: ('--verbose',),
+    }.get(verbosity, ())
+
     command = (
         'attic', 'check',
         repository,
-    ) + _make_check_flags(checks) + (('--verbose',) if verbose else ())
+    ) + _make_check_flags(checks) + verbosity_flags
 
     # Attic's check command spews to stdout even without the verbose flag. Suppress it.
-    stdout = None if verbose else open(os.devnull, 'w')
+    stdout = None if verbosity_flags else open(os.devnull, 'w')
 
     subprocess.check_call(command, stdout=stdout)
diff --git a/atticmatic/command.py b/atticmatic/command.py
index 0a844b20f..37f42be55 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -29,9 +29,9 @@ def parse_arguments(*arguments):
         help='Excludes filename',
     )
     parser.add_argument(
-        '-v', '--verbose',
-        action='store_true',
-        help='Display verbose progress information',
+        '-v', '--verbosity',
+        type=int,
+        help='Display verbose progress (1 for some, 2 for lots)',
     )
 
     return parser.parse_args(arguments)
@@ -43,9 +43,9 @@ def main():
         config = parse_configuration(args.config_filename)
         repository = config.location['repository']
 
-        create_archive(args.excludes_filename, args.verbose, **config.location)
-        prune_archives(args.verbose, repository, config.retention)
-        check_archives(args.verbose, repository, config.consistency)
+        create_archive(args.excludes_filename, args.verbosity, **config.location)
+        prune_archives(args.verbosity, repository, config.retention)
+        check_archives(args.verbosity, repository, config.consistency)
     except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py
index 3a5e7943d..9b7587119 100644
--- a/atticmatic/tests/integration/test_command.py
+++ b/atticmatic/tests/integration/test_command.py
@@ -10,7 +10,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
     assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
-    assert parser.verbose == False
+    assert parser.verbosity == None
 
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
@@ -18,15 +18,15 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
 
     assert parser.config_filename == 'myconfig'
     assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbose == False
+    assert parser.verbosity == None
 
 
-def test_parse_arguments_with_verbose_flag_overrides_default():
-    parser = module.parse_arguments('--verbose')
+def test_parse_arguments_with_verbosity_flag_overrides_default():
+    parser = module.parse_arguments('--verbosity', '1')
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
     assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
-    assert parser.verbose == True
+    assert parser.verbosity == 1
 
 
 def test_parse_arguments_with_invalid_arguments_exits():
diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py
index ab52850c4..13be67b2c 100644
--- a/atticmatic/tests/unit/test_attic.py
+++ b/atticmatic/tests/unit/test_attic.py
@@ -4,6 +4,7 @@ from flexmock import flexmock
 
 from atticmatic import attic as module
 from atticmatic.tests.builtins import builtins_mock
+from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
 def insert_subprocess_mock(check_call_command, **kwargs):
@@ -28,34 +29,43 @@ def insert_datetime_mock():
     ).mock
 
 
+CREATE_COMMAND = ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar')
+
+
 def test_create_archive_should_call_attic_with_parameters():
-    insert_subprocess_mock(
-        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
-    )
+    insert_subprocess_mock(CREATE_COMMAND)
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
         excludes_filename='excludes',
-        verbose=False,
+        verbosity=None,
         source_directories='foo bar',
         repository='repo',
     )
 
 
-def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters():
-    insert_subprocess_mock(
-        (
-            'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar',
-            '--verbose', '--stats',
-        ),
-    )
+def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter():
+    insert_subprocess_mock(CREATE_COMMAND + ('--stats',))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
         excludes_filename='excludes',
-        verbose=True,
+        verbosity=VERBOSITY_SOME,
+        source_directories='foo bar',
+        repository='repo',
+    )
+
+
+def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_parameter():
+    insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=VERBOSITY_LOTS,
         source_directories='foo bar',
         repository='repo',
     )
@@ -82,40 +92,49 @@ def test_make_prune_flags_should_return_flags_from_config():
     assert tuple(result) == BASE_PRUNE_FLAGS
 
 
+PRUNE_COMMAND = (
+    'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
+)
+
+
 def test_prune_archives_should_call_attic_with_parameters():
     retention_config = flexmock()
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_mock(
-        (
-            'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
-            '3',
-        ),
-    )
+    insert_subprocess_mock(PRUNE_COMMAND)
 
     module.prune_archives(
-        verbose=False,
+        verbosity=None,
         repository='repo',
         retention_config=retention_config,
     )
 
 
-def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters():
+def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_parameter():
     retention_config = flexmock()
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_mock(
-        (
-            'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
-            '3', '--verbose',
-        ),
-    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--stats',))
 
     module.prune_archives(
         repository='repo',
-        verbose=True,
+        verbosity=VERBOSITY_SOME,
+        retention_config=retention_config,
+    )
+
+
+def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--verbose', '--stats',))
+
+    module.prune_archives(
+        repository='repo',
+        verbosity=VERBOSITY_LOTS,
         retention_config=retention_config,
     )
 
@@ -171,13 +190,13 @@ def test_check_archives_should_call_attic_with_parameters():
     flexmock(module.os).should_receive('devnull')
 
     module.check_archives(
-        verbose=False,
+        verbosity=None,
         repository='repo',
         consistency_config=consistency_config,
     )
 
 
-def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
+def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
     consistency_config = flexmock()
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
@@ -189,7 +208,25 @@ def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters()
     insert_datetime_mock()
 
     module.check_archives(
-        verbose=True,
+        verbosity=VERBOSITY_SOME,
+        repository='repo',
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    insert_subprocess_mock(
+        ('attic', 'check', 'repo', '--verbose'),
+        stdout=None,
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.check_archives(
+        verbosity=VERBOSITY_LOTS,
         repository='repo',
         consistency_config=consistency_config,
     )
@@ -201,7 +238,7 @@ def test_check_archives_without_any_checks_should_bail():
     insert_subprocess_never()
 
     module.check_archives(
-        verbose=False,
+        verbosity=None,
         repository='repo',
         consistency_config=consistency_config,
     )
diff --git a/atticmatic/verbosity.py b/atticmatic/verbosity.py
new file mode 100644
index 000000000..06dfc4c4e
--- /dev/null
+++ b/atticmatic/verbosity.py
@@ -0,0 +1,2 @@
+VERBOSITY_SOME = 1
+VERBOSITY_LOTS = 2
diff --git a/setup.py b/setup.py
index 6fcd51d7c..2a64c559f 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.6',
+    version='0.0.7',
     description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

From 52d5240fa0e61ca1d9119ef59b24078ecde3bd6e Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Fri, 17 Jul 2015 21:58:58 -0700
Subject: [PATCH 051/189] Added tag 0.0.7 for changeset cf4c7065f071

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 875f102b8..f43c3a7e7 100644
--- a/.hgtags
+++ b/.hgtags
@@ -9,3 +9,4 @@ aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
 569aef47a9b25c55b13753f94706f5d330219995 0.0.5
 a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
 7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
+cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7

From 7097ed67a6b9a3035112dade86ce5cbcd4a778e6 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sat, 18 Jul 2015 18:35:29 -0700
Subject: [PATCH 052/189] New "borgmatic" command to support Borg backup
 software, a fork of Attic.

---
 NEWS                                          |  4 ++
 README.md                                     | 40 +++++++++++------
 atticmatic/backends/__init__.py               |  0
 atticmatic/backends/attic.py                  | 12 +++++
 atticmatic/backends/borg.py                   | 13 ++++++
 atticmatic/{attic.py => backends/shared.py}   | 45 +++++++++++--------
 atticmatic/command.py                         | 40 ++++++++++++-----
 atticmatic/tests/integration/test_command.py  | 21 +++++----
 atticmatic/tests/unit/backends/__init__.py    |  0
 .../test_shared.py}                           | 12 ++++-
 atticmatic/tests/unit/test_command.py         | 33 ++++++++++++++
 sample/config                                 |  2 +-
 setup.py                                      | 11 +++--
 13 files changed, 175 insertions(+), 58 deletions(-)
 create mode 100644 atticmatic/backends/__init__.py
 create mode 100644 atticmatic/backends/attic.py
 create mode 100644 atticmatic/backends/borg.py
 rename atticmatic/{attic.py => backends/shared.py} (74%)
 create mode 100644 atticmatic/tests/unit/backends/__init__.py
 rename atticmatic/tests/unit/{test_attic.py => backends/test_shared.py} (95%)
 create mode 100644 atticmatic/tests/unit/test_command.py

diff --git a/NEWS b/NEWS
index f46598a4f..f8dd0bf11 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.1.0
+
+ * New "borgmatic" command to support Borg backup software, a fork of Attic.
+
 0.0.7
 
  * Flag for multiple levels of verbosity: some, and lots.
diff --git a/README.md b/README.md
index 67b4bc1e4..fe2667604 100644
--- a/README.md
+++ b/README.md
@@ -4,12 +4,13 @@ save_as: atticmatic/index.html
 
 ## Overview
 
-atticmatic is a simple Python wrapper script for the [Attic backup
-software](https://attic-backup.org/) that initiates a backup, prunes any old
-backups according to a retention policy, and validates backups for
-consistency. The script supports specifying your settings in a declarative
-configuration file rather than having to put them all on the command-line, and
-handles common errors.
+atticmatic is a simple Python wrapper script for the
+[Attic](https://attic-backup.org/) and
+[Borg](https://borgbackup.github.io/borgbackup/) backup software that
+initiates a backup, prunes any old backups according to a retention policy,
+and validates backups for consistency. The script supports specifying your
+settings in a declarative configuration file rather than having to put them
+all on the command-line, and handles common errors.
 
 Here's an example config file:
 
@@ -17,7 +18,7 @@ Here's an example config file:
     # Space-separated list of source directories to backup.
     source_directories: /home /etc
 
-    # Path to local or remote Attic repository.
+    # Path to local or remote backup repository.
     repository: user@backupserver:sourcehostname.attic
 
     [retention]
@@ -41,14 +42,14 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on
 
 ## Setup
 
-To get up and running with Attic, follow the [Attic Quick
-Start](https://attic-backup.org/quickstart.html) guide to create an Attic
+To get up and running, follow the [Attic Quick
+Start](https://attic-backup.org/quickstart.html) or the [Borg Quick
+Start](https://borgbackup.github.io/borgbackup/quickstart.html) to create a
 repository on a local or remote host. Note that if you plan to run atticmatic
 on a schedule with cron, and you encrypt your attic repository with a
 passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE`
-environment variable. See [attic's repository encryption
-documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
-more info.
+environment variable. See the repository encryption section of the Quick Start
+for more info.
 
 If the repository is on a remote host, make sure that your local root user has
 key-based ssh access to the desired user account on the remote host.
@@ -57,13 +58,19 @@ To install atticmatic, run the following command to download and install it:
 
     sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
 
-Then copy the following configuration files:
+If you are using Attic, copy the following configuration files:
 
     sudo cp sample/atticmatic.cron /etc/cron.d/atticmatic
     sudo mkdir /etc/atticmatic/
     sudo cp sample/config sample/excludes /etc/atticmatic/
 
-Lastly, modify those files with your desired configuration.
+If you are using Borg, copy the files like this instead:
+
+    sudo cp sample/atticmatic.cron /etc/cron.d/borgmatic
+    sudo mkdir /etc/borgmatic/
+    sudo cp sample/config sample/excludes /etc/borgmatic/
+
+Lastly, modify the /etc files with your desired configuration.
 
 
 ## Usage
@@ -73,6 +80,11 @@ arguments:
 
     atticmatic
 
+Or, if you're using Borg, use this command instead to make use of the Borg
+backend:
+
+    borgmatic
+
 This will also prune any old backups as per the configured retention policy,
 and check backups for consistency problems due to things like file damage.
 
diff --git a/atticmatic/backends/__init__.py b/atticmatic/backends/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/atticmatic/backends/attic.py b/atticmatic/backends/attic.py
new file mode 100644
index 000000000..a99c323b8
--- /dev/null
+++ b/atticmatic/backends/attic.py
@@ -0,0 +1,12 @@
+from functools import partial
+
+from atticmatic.backends import shared
+
+# An atticmatic backend that supports Attic for actually handling backups.
+
+COMMAND = 'attic'
+
+
+create_archive = partial(shared.create_archive, command=COMMAND)
+prune_archives = partial(shared.prune_archives, command=COMMAND)
+check_archives = partial(shared.check_archives, command=COMMAND)
diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py
new file mode 100644
index 000000000..96b5bffb3
--- /dev/null
+++ b/atticmatic/backends/borg.py
@@ -0,0 +1,13 @@
+from functools import partial
+
+from atticmatic.backends import shared
+
+# An atticmatic backend that supports Borg for actually handling backups.
+
+COMMAND = 'borg'
+
+
+create_archive = partial(shared.create_archive, command=COMMAND)
+prune_archives = partial(shared.prune_archives, command=COMMAND)
+check_archives = partial(shared.check_archives, command=COMMAND)
+
diff --git a/atticmatic/attic.py b/atticmatic/backends/shared.py
similarity index 74%
rename from atticmatic/attic.py
rename to atticmatic/backends/shared.py
index 5a0a4b772..eb167b5dc 100644
--- a/atticmatic/attic.py
+++ b/atticmatic/backends/shared.py
@@ -6,10 +6,16 @@ import subprocess
 from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
-def create_archive(excludes_filename, verbosity, source_directories, repository):
+# Common backend functionality shared by Attic and Borg. As the two backup
+# commands diverge, these shared functions will likely need to be replaced
+# with non-shared functions within atticmatic.backends.attic and
+# atticmatic.backends.borg.
+
+
+def create_archive(excludes_filename, verbosity, source_directories, repository, command):
     '''
-    Given an excludes filename, a vebosity flag, a space-separated list of source directories, and
-    a local or remote repository path, create an attic archive.
+    Given an excludes filename, a vebosity flag, a space-separated list of source directories, a
+    local or remote repository path, and a command to run, create an attic archive.
     '''
     sources = tuple(source_directories.split(' '))
     verbosity_flags = {
@@ -17,8 +23,8 @@ def create_archive(excludes_filename, verbosity, source_directories, repository)
         VERBOSITY_LOTS: ('--verbose', '--stats'),
     }.get(verbosity, ())
 
-    command = (
-        'attic', 'create',
+    full_command = (
+        command, 'create',
         '--exclude-from', excludes_filename,
         '{repo}::{hostname}-{timestamp}'.format(
             repo=repository,
@@ -27,7 +33,7 @@ def create_archive(excludes_filename, verbosity, source_directories, repository)
         ),
     ) + sources + verbosity_flags
 
-    subprocess.check_call(command)
+    subprocess.check_call(full_command)
 
 
 def _make_prune_flags(retention_config):
@@ -52,18 +58,19 @@ def _make_prune_flags(retention_config):
     )
 
 
-def prune_archives(verbosity, repository, retention_config):
+def prune_archives(verbosity, repository, retention_config, command):
     '''
-    Given a verbosity flag, a local or remote repository path, and a retention config dict, prune
-    attic archives according the the retention policy specified in that configuration.
+    Given a verbosity flag, a local or remote repository path, a retention config dict, and a
+    command to run, prune attic archives according the the retention policy specified in that
+    configuration.
     '''
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
         VERBOSITY_LOTS: ('--verbose', '--stats'),
     }.get(verbosity, ())
 
-    command = (
-        'attic', 'prune',
+    full_command = (
+        command, 'prune',
         repository,
     ) + tuple(
         element
@@ -71,7 +78,7 @@ def prune_archives(verbosity, repository, retention_config):
         for element in pair
     ) + verbosity_flags
 
-    subprocess.check_call(command)
+    subprocess.check_call(full_command)
 
 
 DEFAULT_CHECKS = ('repository', 'archives')
@@ -123,10 +130,10 @@ def _make_check_flags(checks):
     )
 
 
-def check_archives(verbosity, repository, consistency_config):
+def check_archives(verbosity, repository, consistency_config, command):
     '''
-    Given a verbosity flag, a local or remote repository path, and a consistency config dict, check
-    the contained attic archives for consistency.
+    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
+    command to run, check the contained attic archives for consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
@@ -139,12 +146,12 @@ def check_archives(verbosity, repository, consistency_config):
         VERBOSITY_LOTS: ('--verbose',),
     }.get(verbosity, ())
 
-    command = (
-        'attic', 'check',
+    full_command = (
+        command, 'check',
         repository,
     ) + _make_check_flags(checks) + verbosity_flags
 
-    # Attic's check command spews to stdout even without the verbose flag. Suppress it.
+    # The check command spews to stdout even without the verbose flag. Suppress it.
     stdout = None if verbosity_flags else open(os.devnull, 'w')
 
-    subprocess.check_call(command, stdout=stdout)
+    subprocess.check_call(full_command, stdout=stdout)
diff --git a/atticmatic/command.py b/atticmatic/command.py
index 37f42be55..4578c1eb0 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -1,31 +1,34 @@
 from __future__ import print_function
 from argparse import ArgumentParser
+from importlib import import_module
+import os
 from subprocess import CalledProcessError
 import sys
 
-from atticmatic.attic import check_archives, create_archive, prune_archives
 from atticmatic.config import parse_configuration
 
 
-DEFAULT_CONFIG_FILENAME = '/etc/atticmatic/config'
-DEFAULT_EXCLUDES_FILENAME = '/etc/atticmatic/excludes'
+DEFAULT_CONFIG_FILENAME_PATTERN = '/etc/{}/config'
+DEFAULT_EXCLUDES_FILENAME_PATTERN = '/etc/{}/excludes'
 
 
-def parse_arguments(*arguments):
+def parse_arguments(command_name, *arguments):
     '''
-    Parse the given command-line arguments and return them as an ArgumentParser instance.
+    Given the name of the command with which this script was invoked and command-line arguments,
+    parse the arguments and return them as an ArgumentParser instance. Use the command name to
+    determine the default configuration and excludes paths.
     '''
     parser = ArgumentParser()
     parser.add_argument(
         '-c', '--config',
         dest='config_filename',
-        default=DEFAULT_CONFIG_FILENAME,
+        default=DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name),
         help='Configuration filename',
     )
     parser.add_argument(
         '--excludes',
         dest='excludes_filename',
-        default=DEFAULT_EXCLUDES_FILENAME,
+        default=DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name),
         help='Excludes filename',
     )
     parser.add_argument(
@@ -37,15 +40,30 @@ def parse_arguments(*arguments):
     return parser.parse_args(arguments)
 
 
+def load_backend(command_name):
+    '''
+    Given the name of the command with which this script was invoked, return the corresponding
+    backend module responsible for actually dealing with backups.
+    '''
+    backend_name = {
+        'atticmatic': 'attic',
+        'borgmatic': 'borg',
+    }.get(command_name, 'attic')
+
+    return import_module('atticmatic.backends.{}'.format(backend_name))
+
+
 def main():
     try:
-        args = parse_arguments(*sys.argv[1:])
+        command_name = os.path.basename(sys.argv[0])
+        args = parse_arguments(command_name, *sys.argv[1:])
         config = parse_configuration(args.config_filename)
         repository = config.location['repository']
+        backend = load_backend(command_name)
 
-        create_archive(args.excludes_filename, args.verbosity, **config.location)
-        prune_archives(args.verbosity, repository, config.retention)
-        check_archives(args.verbosity, repository, config.consistency)
+        backend.create_archive(args.excludes_filename, args.verbosity, **config.location)
+        backend.prune_archives(args.verbosity, repository, config.retention)
+        backend.check_archives(args.verbosity, repository, config.consistency)
     except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py
index 9b7587119..fd7259556 100644
--- a/atticmatic/tests/integration/test_command.py
+++ b/atticmatic/tests/integration/test_command.py
@@ -5,16 +5,19 @@ from nose.tools import assert_raises
 from atticmatic import command as module
 
 
-def test_parse_arguments_with_no_arguments_uses_defaults():
-    parser = module.parse_arguments()
+COMMAND_NAME = 'foomatic'
 
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    parser = module.parse_arguments(COMMAND_NAME)
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME)
     assert parser.verbosity == None
 
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
-    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+    parser = module.parse_arguments(COMMAND_NAME, '--config', 'myconfig', '--excludes', 'myexcludes')
 
     assert parser.config_filename == 'myconfig'
     assert parser.excludes_filename == 'myexcludes'
@@ -22,10 +25,10 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
 
 
 def test_parse_arguments_with_verbosity_flag_overrides_default():
-    parser = module.parse_arguments('--verbosity', '1')
+    parser = module.parse_arguments(COMMAND_NAME, '--verbosity', '1')
 
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME)
     assert parser.verbosity == 1
 
 
@@ -35,6 +38,6 @@ def test_parse_arguments_with_invalid_arguments_exits():
 
     try:
         with assert_raises(SystemExit):
-            module.parse_arguments('--posix-me-harder')
+            module.parse_arguments(COMMAND_NAME, '--posix-me-harder')
     finally:
         sys.stderr = original_stderr
diff --git a/atticmatic/tests/unit/backends/__init__.py b/atticmatic/tests/unit/backends/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/backends/test_shared.py
similarity index 95%
rename from atticmatic/tests/unit/test_attic.py
rename to atticmatic/tests/unit/backends/test_shared.py
index 13be67b2c..c779c3660 100644
--- a/atticmatic/tests/unit/test_attic.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -2,7 +2,7 @@ from collections import OrderedDict
 
 from flexmock import flexmock
 
-from atticmatic import attic as module
+from atticmatic.backends import shared as module
 from atticmatic.tests.builtins import builtins_mock
 from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
@@ -42,6 +42,7 @@ def test_create_archive_should_call_attic_with_parameters():
         verbosity=None,
         source_directories='foo bar',
         repository='repo',
+        command='attic',
     )
 
 
@@ -55,6 +56,7 @@ def test_create_archive_with_verbosity_some_should_call_attic_with_stats_paramet
         verbosity=VERBOSITY_SOME,
         source_directories='foo bar',
         repository='repo',
+        command='attic',
     )
 
 
@@ -68,6 +70,7 @@ def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_param
         verbosity=VERBOSITY_LOTS,
         source_directories='foo bar',
         repository='repo',
+        command='attic',
     )
 
 
@@ -108,6 +111,7 @@ def test_prune_archives_should_call_attic_with_parameters():
         verbosity=None,
         repository='repo',
         retention_config=retention_config,
+        command='attic',
     )
 
 
@@ -122,6 +126,7 @@ def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_paramet
         repository='repo',
         verbosity=VERBOSITY_SOME,
         retention_config=retention_config,
+        command='attic',
     )
 
 
@@ -136,6 +141,7 @@ def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_param
         repository='repo',
         verbosity=VERBOSITY_LOTS,
         retention_config=retention_config,
+        command='attic',
     )
 
 
@@ -193,6 +199,7 @@ def test_check_archives_should_call_attic_with_parameters():
         verbosity=None,
         repository='repo',
         consistency_config=consistency_config,
+        command='attic',
     )
 
 
@@ -211,6 +218,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param
         verbosity=VERBOSITY_SOME,
         repository='repo',
         consistency_config=consistency_config,
+        command='attic',
     )
 
 
@@ -229,6 +237,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param
         verbosity=VERBOSITY_LOTS,
         repository='repo',
         consistency_config=consistency_config,
+        command='attic',
     )
 
 
@@ -241,4 +250,5 @@ def test_check_archives_without_any_checks_should_bail():
         verbosity=None,
         repository='repo',
         consistency_config=consistency_config,
+        command='attic',
     )
diff --git a/atticmatic/tests/unit/test_command.py b/atticmatic/tests/unit/test_command.py
new file mode 100644
index 000000000..6a3cdb163
--- /dev/null
+++ b/atticmatic/tests/unit/test_command.py
@@ -0,0 +1,33 @@
+from flexmock import flexmock
+
+from atticmatic import command as module
+
+
+def test_load_backend_with_atticmatic_command_should_return_attic_backend():
+    backend = flexmock()
+    (
+        flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic')
+        .and_return(backend).once()
+    )
+
+    assert module.load_backend('atticmatic') == backend
+
+
+def test_load_backend_with_unknown_command_should_return_attic_backend():
+    backend = flexmock()
+    (
+        flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic')
+        .and_return(backend).once()
+    )
+
+    assert module.load_backend('unknownmatic') == backend
+
+
+def test_load_backend_with_borgmatic_command_should_return_borg_backend():
+    backend = flexmock()
+    (
+        flexmock(module).should_receive('import_module').with_args('atticmatic.backends.borg')
+        .and_return(backend).once()
+    )
+
+    assert module.load_backend('borgmatic') == backend
diff --git a/sample/config b/sample/config
index b13afbe60..6ae08b457 100644
--- a/sample/config
+++ b/sample/config
@@ -2,7 +2,7 @@
 # Space-separated list of source directories to backup.
 source_directories: /home /etc
 
-# Path to local or remote Attic repository.
+# Path to local or remote repository.
 repository: user@backupserver:sourcehostname.attic
 
 [retention]
diff --git a/setup.py b/setup.py
index 2a64c559f..5736ccc07 100644
--- a/setup.py
+++ b/setup.py
@@ -2,12 +2,17 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.7',
-    description='A wrapper script for Attic backup software that creates and prunes backups',
+    version='0.1.0',
+    description='A wrapper script for Attic/Borg backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',
     packages=find_packages(),
-    entry_points={'console_scripts': ['atticmatic = atticmatic.command:main']},
+    entry_points={
+        'console_scripts': [
+            'atticmatic = atticmatic.command:main',
+            'borgmatic = atticmatic.command:main',
+        ]
+    },
     tests_require=(
         'flexmock',
         'nose',

From d25db4cd0dda7677f93ba26b324a8d16d87d62af Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sat, 18 Jul 2015 18:39:33 -0700
Subject: [PATCH 053/189] Added tag 0.1.0 for changeset 38d72677343f

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index f43c3a7e7..118bf8097 100644
--- a/.hgtags
+++ b/.hgtags
@@ -10,3 +10,4 @@ aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
 a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
 7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
 cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
+38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0

From 6b0aa13856835e1a415c01fe66df8faa533939a1 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sat, 18 Jul 2015 18:44:11 -0700
Subject: [PATCH 054/189] Adding borgmatic cron example.

---
 NEWS                  | 4 ++++
 README.md             | 2 +-
 sample/borgmatic.cron | 3 +++
 3 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 sample/borgmatic.cron

diff --git a/NEWS b/NEWS
index f8dd0bf11..159c71c69 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.1.1
+
+ * Adding borgmatic cron example, and updating documentation to refer to it.
+
 0.1.0
 
  * New "borgmatic" command to support Borg backup software, a fork of Attic.
diff --git a/README.md b/README.md
index fe2667604..69f18c01c 100644
--- a/README.md
+++ b/README.md
@@ -66,7 +66,7 @@ If you are using Attic, copy the following configuration files:
 
 If you are using Borg, copy the files like this instead:
 
-    sudo cp sample/atticmatic.cron /etc/cron.d/borgmatic
+    sudo cp sample/borgmatic.cron /etc/cron.d/borgmatic
     sudo mkdir /etc/borgmatic/
     sudo cp sample/config sample/excludes /etc/borgmatic/
 
diff --git a/sample/borgmatic.cron b/sample/borgmatic.cron
new file mode 100644
index 000000000..2f6f9121e
--- /dev/null
+++ b/sample/borgmatic.cron
@@ -0,0 +1,3 @@
+# You can drop this file into /etc/cron.d/ to run borgmatic nightly.
+
+0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/borgmatic

From ce6196a5c6f62607629d952614c511fceb67cdbd Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sat, 18 Jul 2015 18:44:14 -0700
Subject: [PATCH 055/189] Added tag 0.1.1 for changeset ac5dfa01e9d1

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 118bf8097..642f1c3b4 100644
--- a/.hgtags
+++ b/.hgtags
@@ -11,3 +11,4 @@ a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
 7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
 cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
 38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0
+ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1

From 1f1c8fdabacdd2a8b896fa1219be5a6de86a0820 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 18 Jul 2015 23:48:58 -0700
Subject: [PATCH 056/189] Bumping version.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 5736ccc07..7d535bca6 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.1.0',
+    version='0.1.1',
     description='A wrapper script for Attic/Borg backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

From 17ac63aae6e63a4ea3df55dca4cdd51f882271b2 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 18 Jul 2015 23:49:06 -0700
Subject: [PATCH 057/189] Added tag 0.1.1 for changeset 7b6c87dca7ea

---
 .hgtags | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.hgtags b/.hgtags
index 642f1c3b4..cc4911587 100644
--- a/.hgtags
+++ b/.hgtags
@@ -12,3 +12,5 @@ a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
 cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
 38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0
 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
+ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
+7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1

From 52ab7cb881cbd9f3e3c43cb30cfe5fc5d04b9b5e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 21 Jul 2015 21:29:40 -0700
Subject: [PATCH 058/189] New issue tracker, linked from documentation.

---
 NEWS      | 4 ++++
 README.md | 7 +++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 159c71c69..5f6243cc1 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.1.2-dev
+
+ * New issue tracker, linked from documentation.
+
 0.1.1
 
  * Adding borgmatic cron example, and updating documentation to refer to it.
diff --git a/README.md b/README.md
index 69f18c01c..b738c9644 100644
--- a/README.md
+++ b/README.md
@@ -135,6 +135,9 @@ This should make the client keep the connection alive while validating
 backups.
 
 
-## Feedback
+## Issues and feedback
 
-Questions? Comments? Got a patch? Contact <mailto:witten@torsion.org>.
+Got an issue or an idea for a feature enhancement? Check out the [atticmatic
+issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues).
+
+Other questions or comments? Contact <mailto:witten@torsion.org>.

From 38322a3f6fe71f9bbd0077cb628388ff36c98a6a Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sun, 26 Jul 2015 20:57:31 -0700
Subject: [PATCH 059/189] Linking to both Attic and Borg prune docs from sample
 config.

---
 sample/config | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sample/config b/sample/config
index 6ae08b457..4e09b04e7 100644
--- a/sample/config
+++ b/sample/config
@@ -7,7 +7,8 @@ repository: user@backupserver:sourcehostname.attic
 
 [retention]
 # Retention policy for how many backups to keep in each category. See
-# https://attic-backup.org/usage.html#attic-prune for details.
+# https://attic-backup.org/usage.html#attic-prune or
+# https://borgbackup.github.io/borgbackup/usage.html#borg-prune for details.
 #keep_within: 3h
 #keep_hourly: 24
 keep_daily: 7

From 58d33503a1bfbe4e44332d388a7b0dc63099e7dc Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sun, 26 Jul 2015 21:06:03 -0700
Subject: [PATCH 060/189] As a convenience to new users, allow a missing
 default excludes file.

---
 NEWS                                          |  3 +-
 atticmatic/backends/shared.py                 |  8 ++---
 atticmatic/command.py                         |  7 +++--
 atticmatic/tests/integration/test_command.py  | 29 +++++++++++++++++++
 atticmatic/tests/unit/backends/test_shared.py | 17 ++++++++++-
 5 files changed, 56 insertions(+), 8 deletions(-)

diff --git a/NEWS b/NEWS
index 5f6243cc1..db51ca399 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
-0.1.2-dev
+0.1.2
 
+ * As a convenience to new users, allow a missing default excludes file.
  * New issue tracker, linked from documentation.
 
 0.1.1
diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index eb167b5dc..8abd064b9 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -14,10 +14,11 @@ from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 def create_archive(excludes_filename, verbosity, source_directories, repository, command):
     '''
-    Given an excludes filename, a vebosity flag, a space-separated list of source directories, a
-    local or remote repository path, and a command to run, create an attic archive.
+    Given an excludes filename (or None), a vebosity flag, a space-separated list of source
+    directories, a local or remote repository path, and a command to run, create an attic archive.
     '''
     sources = tuple(source_directories.split(' '))
+    exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
         VERBOSITY_LOTS: ('--verbose', '--stats'),
@@ -25,13 +26,12 @@ def create_archive(excludes_filename, verbosity, source_directories, repository,
 
     full_command = (
         command, 'create',
-        '--exclude-from', excludes_filename,
         '{repo}::{hostname}-{timestamp}'.format(
             repo=repository,
             hostname=platform.node(),
             timestamp=datetime.now().isoformat(),
         ),
-    ) + sources + verbosity_flags
+    ) + sources + exclude_flags + verbosity_flags
 
     subprocess.check_call(full_command)
 
diff --git a/atticmatic/command.py b/atticmatic/command.py
index 4578c1eb0..13e7b4346 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -18,17 +18,20 @@ def parse_arguments(command_name, *arguments):
     parse the arguments and return them as an ArgumentParser instance. Use the command name to
     determine the default configuration and excludes paths.
     '''
+    config_filename_default = DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name)
+    excludes_filename_default = DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name)
+
     parser = ArgumentParser()
     parser.add_argument(
         '-c', '--config',
         dest='config_filename',
-        default=DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name),
+        default=config_filename_default,
         help='Configuration filename',
     )
     parser.add_argument(
         '--excludes',
         dest='excludes_filename',
-        default=DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name),
+        default=excludes_filename_default if os.path.exists(excludes_filename_default) else None,
         help='Excludes filename',
     )
     parser.add_argument(
diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py
index fd7259556..346612bf6 100644
--- a/atticmatic/tests/integration/test_command.py
+++ b/atticmatic/tests/integration/test_command.py
@@ -1,5 +1,7 @@
+import os
 import sys
 
+from flexmock import flexmock
 from nose.tools import assert_raises
 
 from atticmatic import command as module
@@ -9,6 +11,8 @@ COMMAND_NAME = 'foomatic'
 
 
 def test_parse_arguments_with_no_arguments_uses_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
     parser = module.parse_arguments(COMMAND_NAME)
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
@@ -17,6 +21,8 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
 
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
     parser = module.parse_arguments(COMMAND_NAME, '--config', 'myconfig', '--excludes', 'myexcludes')
 
     assert parser.config_filename == 'myconfig'
@@ -24,7 +30,29 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
     assert parser.verbosity == None
 
 
+def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
+    flexmock(os.path).should_receive('exists').and_return(False)
+
+    parser = module.parse_arguments(COMMAND_NAME)
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.excludes_filename == None
+    assert parser.verbosity == None
+
+
+def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
+    flexmock(os.path).should_receive('exists').and_return(False)
+
+    parser = module.parse_arguments(COMMAND_NAME, '--excludes', 'myexcludes')
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.excludes_filename == 'myexcludes'
+    assert parser.verbosity == None
+
+
 def test_parse_arguments_with_verbosity_flag_overrides_default():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
     parser = module.parse_arguments(COMMAND_NAME, '--verbosity', '1')
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
@@ -33,6 +61,7 @@ def test_parse_arguments_with_verbosity_flag_overrides_default():
 
 
 def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(os.path).should_receive('exists').and_return(True)
     original_stderr = sys.stderr
     sys.stderr = sys.stdout
 
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index c779c3660..8fe236f23 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -29,7 +29,8 @@ def insert_datetime_mock():
     ).mock
 
 
-CREATE_COMMAND = ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar')
+CREATE_COMMAND_WITHOUT_EXCLUDES = ('attic', 'create', 'repo::host-now', 'foo', 'bar')
+CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes')
 
 
 def test_create_archive_should_call_attic_with_parameters():
@@ -46,6 +47,20 @@ def test_create_archive_should_call_attic_with_parameters():
     )
 
 
+def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes():
+    insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
 def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter():
     insert_subprocess_mock(CREATE_COMMAND + ('--stats',))
     insert_platform_mock()

From c27b4a34976ba078857d61c47cb3d2f2befb9758 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sun, 26 Jul 2015 21:06:06 -0700
Subject: [PATCH 061/189] Added tag 0.1.2 for changeset 83067f995dd3

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index cc4911587..005ecdf25 100644
--- a/.hgtags
+++ b/.hgtags
@@ -14,3 +14,4 @@ cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
 7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1
+83067f995dd391e38544a7722dc3b254b59c5521 0.1.2

From f94181480cefb6d8bf073d8b30b3c1538ef5795d Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sun, 26 Jul 2015 21:29:14 -0700
Subject: [PATCH 062/189] Removing some annoying Pelican metadata from docs.

---
 README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.md b/README.md
index b738c9644..ce38d8003 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,4 @@
 title: Atticmatic
-date:
-save_as: atticmatic/index.html
 
 ## Overview
 

From 952a691f60b6d257be2d203dc51c9472754049f8 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Sun, 26 Jul 2015 22:02:43 -0700
Subject: [PATCH 063/189] Linking to both Attic and Borg check docs from sample
 config.

---
 atticmatic/backends/borg.py | 1 -
 sample/config               | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py
index 96b5bffb3..5214ca1f5 100644
--- a/atticmatic/backends/borg.py
+++ b/atticmatic/backends/borg.py
@@ -10,4 +10,3 @@ COMMAND = 'borg'
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)
 check_archives = partial(shared.check_archives, command=COMMAND)
-
diff --git a/sample/config b/sample/config
index 4e09b04e7..7ee525625 100644
--- a/sample/config
+++ b/sample/config
@@ -20,5 +20,6 @@ keep_yearly: 1
 [consistency]
 # Space-separated list of consistency checks to run: "repository", "archives",
 # or both. Defaults to both. Set to "disabled" to disable all consistency
-# checks. See https://attic-backup.org/usage.html#attic-check for details.
+# checks. See https://attic-backup.org/usage.html#attic-check or
+# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details.
 checks: repository archives

From f5e0e10143a6dcc1a36a535b34b7420af819b0d4 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Mon, 27 Jul 2015 19:06:39 -0700
Subject: [PATCH 064/189] #6: Fixing example config file to use valid
 keep_within value.

---
 sample/config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sample/config b/sample/config
index 7ee525625..c83c554e8 100644
--- a/sample/config
+++ b/sample/config
@@ -9,7 +9,7 @@ repository: user@backupserver:sourcehostname.attic
 # Retention policy for how many backups to keep in each category. See
 # https://attic-backup.org/usage.html#attic-prune or
 # https://borgbackup.github.io/borgbackup/usage.html#borg-prune for details.
-#keep_within: 3h
+#keep_within: 3H
 #keep_hourly: 24
 keep_daily: 7
 keep_weekly: 4

From 9c0687407311e1d01bc9ee08f7cc3f1a12d68719 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Mon, 27 Jul 2015 21:47:52 -0700
Subject: [PATCH 065/189] #1: Add support for "borg check --last N" to Borg
 backend.

---
 NEWS                                          |  4 ++
 atticmatic/backends/attic.py                  |  2 +-
 atticmatic/backends/borg.py                   | 12 +++++
 atticmatic/backends/shared.py                 | 42 ++++++++++++++--
 atticmatic/command.py                         |  4 +-
 atticmatic/config.py                          | 48 ++++---------------
 atticmatic/tests/unit/backends/test_shared.py | 26 +++++++---
 atticmatic/tests/unit/test_config.py          | 11 +++--
 sample/config                                 |  2 +
 9 files changed, 94 insertions(+), 57 deletions(-)

diff --git a/NEWS b/NEWS
index db51ca399..207769e4f 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.1.3
+
+ * #1: Add support for "borg check --last N" to Borg backend.
+
 0.1.2
 
  * As a convenience to new users, allow a missing default excludes file.
diff --git a/atticmatic/backends/attic.py b/atticmatic/backends/attic.py
index a99c323b8..fcacce31e 100644
--- a/atticmatic/backends/attic.py
+++ b/atticmatic/backends/attic.py
@@ -5,7 +5,7 @@ from atticmatic.backends import shared
 # An atticmatic backend that supports Attic for actually handling backups.
 
 COMMAND = 'attic'
-
+CONFIG_FORMAT = shared.CONFIG_FORMAT
 
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)
diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py
index 5214ca1f5..bd6a38693 100644
--- a/atticmatic/backends/borg.py
+++ b/atticmatic/backends/borg.py
@@ -1,10 +1,22 @@
 from functools import partial
 
+from atticmatic.config import Section_format, option
 from atticmatic.backends import shared
 
 # An atticmatic backend that supports Borg for actually handling backups.
 
 COMMAND = 'borg'
+CONFIG_FORMAT = (
+    shared.CONFIG_FORMAT[0],  # location
+    shared.CONFIG_FORMAT[1],  # retention
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+            option('check_last', required=False),
+        ),
+    )
+)
 
 
 create_archive = partial(shared.create_archive, command=COMMAND)
diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index 8abd064b9..74f553366 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -3,6 +3,7 @@ import os
 import platform
 import subprocess
 
+from atticmatic.config import Section_format, option
 from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
@@ -12,6 +13,34 @@ from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 # atticmatic.backends.borg.
 
 
+CONFIG_FORMAT = (
+    Section_format(
+        'location',
+        (
+            option('source_directories'),
+            option('repository'),
+        ),
+    ),
+    Section_format(
+        'retention',
+        (
+            option('keep_within', required=False),
+            option('keep_hourly', int, required=False),
+            option('keep_daily', int, required=False),
+            option('keep_weekly', int, required=False),
+            option('keep_monthly', int, required=False),
+            option('keep_yearly', int, required=False),
+            option('prefix', required=False),
+        ),
+    ),
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+        ),
+    )
+)
+
 def create_archive(excludes_filename, verbosity, source_directories, repository, command):
     '''
     Given an excludes filename (or None), a vebosity flag, a space-separated list of source
@@ -110,7 +139,7 @@ def _parse_checks(consistency_config):
     )
 
 
-def _make_check_flags(checks):
+def _make_check_flags(checks, check_last=None):
     '''
     Given a parsed sequence of checks, transform it into tuple of command-line flags.
 
@@ -121,13 +150,17 @@ def _make_check_flags(checks):
     This will be returned as:
     
         ('--repository-only',)
+
+    Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
+    Borg supports this flag.
     '''
+    last_flag = ('--last', check_last) if check_last else ()
     if checks == DEFAULT_CHECKS:
-        return ()
+        return last_flag
 
     return tuple(
         '--{}-only'.format(check) for check in checks
-    )
+    ) + last_flag
 
 
 def check_archives(verbosity, repository, consistency_config, command):
@@ -138,6 +171,7 @@ def check_archives(verbosity, repository, consistency_config, command):
     If there are no consistency checks to run, skip running them.
     '''
     checks = _parse_checks(consistency_config)
+    check_last = consistency_config.get('check_last', None)
     if not checks:
         return
 
@@ -149,7 +183,7 @@ def check_archives(verbosity, repository, consistency_config, command):
     full_command = (
         command, 'check',
         repository,
-    ) + _make_check_flags(checks) + verbosity_flags
+    ) + _make_check_flags(checks, check_last) + verbosity_flags
 
     # The check command spews to stdout even without the verbose flag. Suppress it.
     stdout = None if verbosity_flags else open(os.devnull, 'w')
diff --git a/atticmatic/command.py b/atticmatic/command.py
index 13e7b4346..0f512e106 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -60,9 +60,9 @@ def main():
     try:
         command_name = os.path.basename(sys.argv[0])
         args = parse_arguments(command_name, *sys.argv[1:])
-        config = parse_configuration(args.config_filename)
-        repository = config.location['repository']
         backend = load_backend(command_name)
+        config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT)
+        repository = config.location['repository']
 
         backend.create_archive(args.excludes_filename, args.verbosity, **config.location)
         backend.prune_archives(args.verbosity, repository, config.retention)
diff --git a/atticmatic/config.py b/atticmatic/config.py
index 7474f0502..bc1a32d16 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -20,35 +20,6 @@ def option(name, value_type=str, required=True):
     return Config_option(name, value_type, required)
 
 
-CONFIG_FORMAT = (
-    Section_format(
-        'location',
-        (
-            option('source_directories'),
-            option('repository'),
-        ),
-    ),
-    Section_format(
-        'retention',
-        (
-            option('keep_within', required=False),
-            option('keep_hourly', int, required=False),
-            option('keep_daily', int, required=False),
-            option('keep_weekly', int, required=False),
-            option('keep_monthly', int, required=False),
-            option('keep_yearly', int, required=False),
-            option('prefix', required=False),
-        ),
-    ),
-    Section_format(
-        'consistency',
-        (
-            option('checks', required=False),
-        ),
-    )
-)
-
-
 def validate_configuration_format(parser, config_format):
     '''
     Given an open ConfigParser and an expected config file format, validate that the parsed
@@ -110,11 +81,6 @@ def validate_configuration_format(parser, config_format):
             )
 
 
-# Describes a parsed configuration, where each attribute is the name of a configuration file section 
-# and each value is a dict of that section's parsed options.
-Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
-
-
 def parse_section_options(parser, section_format):
     '''
     Given an open ConfigParser and an expected section format, return the option values from that
@@ -135,21 +101,25 @@ def parse_section_options(parser, section_format):
     )
 
 
-def parse_configuration(config_filename):
+def parse_configuration(config_filename, config_format):
     '''
-    Given a config filename of the expected format, return the parsed configuration as Parsed_config
-    data structure.
+    Given a config filename and an expected config file format, return the parsed configuration
+    as a namedtuple with one attribute for each parsed section.
 
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
     parser = ConfigParser()
     parser.read(config_filename)
 
-    validate_configuration_format(parser, CONFIG_FORMAT)
+    validate_configuration_format(parser, config_format)
+
+    # Describes a parsed configuration, where each attribute is the name of a configuration file
+    # section and each value is a dict of that section's parsed options.
+    Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format))
 
     return Parsed_config(
         *(
             parse_section_options(parser, section_format)
-            for section_format in CONFIG_FORMAT
+            for section_format in config_format
         )
     )
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index 8fe236f23..c742087d1 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -196,10 +196,24 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
     assert flags == ()
 
 
+def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
+    flags = module._make_check_flags(('foo', 'bar'), check_last=3)
+
+    assert flags == ('--foo-only', '--bar-only', '--last', 3)
+
+
+def test_make_check_flags_with_last_returns_last_flag():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
+
+    assert flags == ('--last', 3)
+
+
 def test_check_archives_should_call_attic_with_parameters():
-    consistency_config = flexmock()
-    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
-    flexmock(module).should_receive('_make_check_flags').and_return(())
+    checks = flexmock()
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
     stdout = flexmock()
     insert_subprocess_mock(
         ('attic', 'check', 'repo'),
@@ -219,7 +233,7 @@ def test_check_archives_should_call_attic_with_parameters():
 
 
 def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
-    consistency_config = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
@@ -238,7 +252,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param
 
 
 def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
-    consistency_config = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
@@ -257,7 +271,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param
 
 
 def test_check_archives_without_any_checks_should_bail():
-    consistency_config = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(())
     insert_subprocess_never()
 
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index 02f1f521b..4c889b091 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -205,17 +205,18 @@ def insert_mock_parser():
 
 def test_parse_configuration_should_return_section_configs():
     parser = insert_mock_parser()
+    config_format = (flexmock(name='items'), flexmock(name='things'))
     mock_module = flexmock(module)
     mock_module.should_receive('validate_configuration_format').with_args(
-        parser, module.CONFIG_FORMAT,
+        parser, config_format,
     ).once()
-    mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT)
+    mock_section_configs = (flexmock(), flexmock())
 
-    for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
+    for section_format, section_config in zip(config_format, mock_section_configs):
         mock_module.should_receive('parse_section_options').with_args(
             parser, section_format,
         ).and_return(section_config).once()
 
-    parsed_config = module.parse_configuration('filename')
+    parsed_config = module.parse_configuration('filename', config_format)
 
-    assert parsed_config == module.Parsed_config(*mock_section_configs)
+    assert parsed_config == type(parsed_config)(*mock_section_configs)
diff --git a/sample/config b/sample/config
index c83c554e8..82d77d12b 100644
--- a/sample/config
+++ b/sample/config
@@ -23,3 +23,5 @@ keep_yearly: 1
 # checks. See https://attic-backup.org/usage.html#attic-check or
 # https://borgbackup.github.io/borgbackup/usage.html#borg-check for details.
 checks: repository archives
+# For Borg only, you can restrict the number of checked archives to the last n.
+#check_last: 3

From e996e09657becc1a936314254aa102143eeeb5d4 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@energysavvy.com>
Date: Mon, 27 Jul 2015 21:48:21 -0700
Subject: [PATCH 066/189] Added tag 0.1.3 for changeset acc7fb61566f

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 005ecdf25..e4e3ada2d 100644
--- a/.hgtags
+++ b/.hgtags
@@ -15,3 +15,4 @@ ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
 7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1
 83067f995dd391e38544a7722dc3b254b59c5521 0.1.2
+acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3

From d041e23d35ddea84cff390bc80160af1284e8775 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Thu, 30 Jul 2015 08:12:31 -0700
Subject: [PATCH 067/189] Adding test that setup.py version matches release
 version.

---
 NEWS     | 4 ++++
 setup.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 207769e4f..367281eba 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.1.4
+
+ * Adding test that setup.py version matches release version.
+
 0.1.3
 
  * #1: Add support for "borg check --last N" to Borg backend.
diff --git a/setup.py b/setup.py
index 7d535bca6..5aee184cd 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.1.1',
+    version='0.1.4',
     description='A wrapper script for Attic/Borg backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

From 61969d17a262f354d14083ec9e582474d336b67a Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Thu, 30 Jul 2015 08:12:36 -0700
Subject: [PATCH 068/189] Added tag 0.1.4 for changeset 6dda59c12de8

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index e4e3ada2d..e8f26b924 100644
--- a/.hgtags
+++ b/.hgtags
@@ -16,3 +16,4 @@ ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
 7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1
 83067f995dd391e38544a7722dc3b254b59c5521 0.1.2
 acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
+6dda59c12de88f060eb7244e6d330173985a9639 0.1.4

From 996ca19dac815b061e396a78e9a6cdda136e5246 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Thu, 30 Jul 2015 08:13:27 -0700
Subject: [PATCH 069/189] Adding version test.

---
 atticmatic/tests/integration/test_version.py | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 atticmatic/tests/integration/test_version.py

diff --git a/atticmatic/tests/integration/test_version.py b/atticmatic/tests/integration/test_version.py
new file mode 100644
index 000000000..e5ba4861f
--- /dev/null
+++ b/atticmatic/tests/integration/test_version.py
@@ -0,0 +1,8 @@
+import subprocess
+
+
+def test_setup_version_matches_news_version():
+    setup_version = subprocess.check_output(('python', 'setup.py', '--version')).decode('ascii')
+    news_version = open('NEWS').readline()
+
+    assert setup_version == news_version

From 1334da99e2b571ed897e104c658d490e7a7ef516 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Thu, 30 Jul 2015 08:13:32 -0700
Subject: [PATCH 070/189] Added tag 0.1.4 for changeset e58246fc92bb

---
 .hgtags | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.hgtags b/.hgtags
index e8f26b924..844f466aa 100644
--- a/.hgtags
+++ b/.hgtags
@@ -17,3 +17,5 @@ ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
 83067f995dd391e38544a7722dc3b254b59c5521 0.1.2
 acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
 6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
+6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
+e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4

From 204e515bf7b3ba16ff72e3dc26d3d58b6136eeb2 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 9 Aug 2015 10:59:27 -0700
Subject: [PATCH 071/189] Changes to support release on PyPI. Now pip
 installable by name!

---
 .hgignore |  1 +
 NEWS      |  4 ++++
 README.md |  2 +-
 setup.cfg |  3 +++
 setup.py  | 17 ++++++++++++++++-
 5 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/.hgignore b/.hgignore
index d7c266a54..d814e3812 100644
--- a/.hgignore
+++ b/.hgignore
@@ -3,3 +3,4 @@ syntax: glob
 *.pyc
 *.swp
 .tox
+dist
diff --git a/NEWS b/NEWS
index 367281eba..482da6b65 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.1.5
+
+ * Changes to support release on PyPI. Now pip installable by name!
+
 0.1.4
 
  * Adding test that setup.py version matches release version.
diff --git a/README.md b/README.md
index ce38d8003..8ed8548ce 100644
--- a/README.md
+++ b/README.md
@@ -54,7 +54,7 @@ key-based ssh access to the desired user account on the remote host.
 
 To install atticmatic, run the following command to download and install it:
 
-    sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
+    sudo pip install --upgrade atticmatic
 
 If you are using Attic, copy the following configuration files:
 
diff --git a/setup.cfg b/setup.cfg
index ead29d1ca..222a3da98 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,5 @@
+[metadata]
+description-file=README.md
+
 [nosetests]
 detailed-errors=1
diff --git a/setup.py b/setup.py
index 5aee184cd..ad32a1458 100644
--- a/setup.py
+++ b/setup.py
@@ -1,11 +1,26 @@
 from setuptools import setup, find_packages
 
+
+VERSION = '0.1.5'
+
+
 setup(
     name='atticmatic',
-    version='0.1.4',
+    version=VERSION,
     description='A wrapper script for Attic/Borg backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',
+    url='https://torsion.org/atticmatic',
+    download_url='https://torsion.org/hg/atticmatic/archive/%s.tar.gz' % VERSION,
+    classifiers=(
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+        'Programming Language :: Python',
+        'Topic :: Security :: Cryptography',
+        'Topic :: System :: Archiving :: Backup',
+    ),
     packages=find_packages(),
     entry_points={
         'console_scripts': [

From 5299046b6bb95134185836fa04e093ba5854b612 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 9 Aug 2015 10:59:40 -0700
Subject: [PATCH 072/189] Added tag 0.1.5 for changeset 0afff209b902

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 844f466aa..076d808e4 100644
--- a/.hgtags
+++ b/.hgtags
@@ -19,3 +19,4 @@ acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
 6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
 6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
 e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
+0afff209b902698c2266986129d6dc9f5f913101 0.1.5

From c67ab09e4d6f61b26605336bcecb870c62c7b555 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 9 Aug 2015 11:04:57 -0700
Subject: [PATCH 073/189] Adding build to hgignore.

---
 .hgignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgignore b/.hgignore
index d814e3812..41280764d 100644
--- a/.hgignore
+++ b/.hgignore
@@ -3,4 +3,5 @@ syntax: glob
 *.pyc
 *.swp
 .tox
+build
 dist

From 30f6ec4f7dee9d358c9dd62287c9b6c705977f8a Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Wed, 2 Sep 2015 18:45:15 -0700
Subject: [PATCH 074/189] Adding documentation note about logging into the
 issue tracker in order to create issues.

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8ed8548ce..246b86a87 100644
--- a/README.md
+++ b/README.md
@@ -136,6 +136,8 @@ backups.
 ## Issues and feedback
 
 Got an issue or an idea for a feature enhancement? Check out the [atticmatic
-issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues).
+issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). In
+order to create a new issue or comment on an issue, you'll need to [login
+first](https://tree.taiga.io/login).
 
 Other questions or comments? Contact <mailto:witten@torsion.org>.

From 3a9e32a4119a0a26b2c4679e22f13af56700c4a6 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Wed, 2 Sep 2015 22:48:07 -0700
Subject: [PATCH 075/189] #9: New configuration option for the encryption
 passphrase. #10: Support for Borg's new archive compression feature.

---
 NEWS                                          |  5 +++
 README.md                                     |  6 +--
 atticmatic/backends/attic.py                  |  2 +
 atticmatic/backends/borg.py                   | 10 ++++-
 atticmatic/backends/shared.py                 | 27 ++++++++++--
 atticmatic/command.py                         |  5 ++-
 atticmatic/tests/unit/backends/test_shared.py | 42 +++++++++++++++++++
 sample/config                                 |  9 ++++
 setup.py                                      |  2 +-
 9 files changed, 98 insertions(+), 10 deletions(-)

diff --git a/NEWS b/NEWS
index 482da6b65..d5185023b 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+0.1.6
+
+ * #9: New configuration option for the encryption passphrase.
+ * #10: Support for Borg's new archive compression feature.
+
 0.1.5
 
  * Changes to support release on PyPI. Now pip installable by name!
diff --git a/README.md b/README.md
index 246b86a87..3d9d9991a 100644
--- a/README.md
+++ b/README.md
@@ -45,9 +45,9 @@ Start](https://attic-backup.org/quickstart.html) or the [Borg Quick
 Start](https://borgbackup.github.io/borgbackup/quickstart.html) to create a
 repository on a local or remote host. Note that if you plan to run atticmatic
 on a schedule with cron, and you encrypt your attic repository with a
-passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE`
-environment variable. See the repository encryption section of the Quick Start
-for more info.
+passphrase instead of a key file, you'll need to set the atticmatic
+`encryption_passphrase` configuration variable. See the repository encryption
+section of the Quick Start for more info.
 
 If the repository is on a remote host, make sure that your local root user has
 key-based ssh access to the desired user account on the remote host.
diff --git a/atticmatic/backends/attic.py b/atticmatic/backends/attic.py
index fcacce31e..daa0b2967 100644
--- a/atticmatic/backends/attic.py
+++ b/atticmatic/backends/attic.py
@@ -7,6 +7,8 @@ from atticmatic.backends import shared
 COMMAND = 'attic'
 CONFIG_FORMAT = shared.CONFIG_FORMAT
 
+
+initialize = partial(shared.initialize, command=COMMAND)
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)
 check_archives = partial(shared.check_archives, command=COMMAND)
diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py
index bd6a38693..222c7028f 100644
--- a/atticmatic/backends/borg.py
+++ b/atticmatic/backends/borg.py
@@ -8,7 +8,14 @@ from atticmatic.backends import shared
 COMMAND = 'borg'
 CONFIG_FORMAT = (
     shared.CONFIG_FORMAT[0],  # location
-    shared.CONFIG_FORMAT[1],  # retention
+    Section_format(
+        'storage',
+        (
+            option('encryption_passphrase', required=False),
+            option('compression', required=False),
+        ),
+    ),
+    shared.CONFIG_FORMAT[2],  # retention
     Section_format(
         'consistency',
         (
@@ -19,6 +26,7 @@ CONFIG_FORMAT = (
 )
 
 
+initialize = partial(shared.initialize, command=COMMAND)
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)
 check_archives = partial(shared.check_archives, command=COMMAND)
diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index 74f553366..5baff3a92 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -21,6 +21,12 @@ CONFIG_FORMAT = (
             option('repository'),
         ),
     ),
+    Section_format(
+        'storage',
+        (
+            option('encryption_passphrase', required=False),
+        ),
+    ),
     Section_format(
         'retention',
         (
@@ -41,13 +47,26 @@ CONFIG_FORMAT = (
     )
 )
 
-def create_archive(excludes_filename, verbosity, source_directories, repository, command):
+
+def initialize(storage_config, command):
+    passphrase = storage_config.get('encryption_passphrase')
+
+    if passphrase:
+        os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase
+
+
+def create_archive(
+    excludes_filename, verbosity, storage_config, source_directories, repository, command,
+):
     '''
-    Given an excludes filename (or None), a vebosity flag, a space-separated list of source
-    directories, a local or remote repository path, and a command to run, create an attic archive.
+    Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
+    list of source directories, a local or remote repository path, and a command to run, create an
+    attic archive.
     '''
     sources = tuple(source_directories.split(' '))
     exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
+    compression = storage_config.get('compression', None)
+    compression_flags = ('--compression', compression) if compression else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
         VERBOSITY_LOTS: ('--verbose', '--stats'),
@@ -60,7 +79,7 @@ def create_archive(excludes_filename, verbosity, source_directories, repository,
             hostname=platform.node(),
             timestamp=datetime.now().isoformat(),
         ),
-    ) + sources + exclude_flags + verbosity_flags
+    ) + sources + exclude_flags + compression_flags + verbosity_flags
 
     subprocess.check_call(full_command)
 
diff --git a/atticmatic/command.py b/atticmatic/command.py
index 0f512e106..08ea49e8d 100644
--- a/atticmatic/command.py
+++ b/atticmatic/command.py
@@ -64,7 +64,10 @@ def main():
         config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT)
         repository = config.location['repository']
 
-        backend.create_archive(args.excludes_filename, args.verbosity, **config.location)
+        backend.initialize(config.storage)
+        backend.create_archive(
+            args.excludes_filename, args.verbosity, config.storage, **config.location
+        )
         backend.prune_archives(args.verbosity, repository, config.retention)
         backend.check_archives(args.verbosity, repository, config.consistency)
     except (ValueError, IOError, CalledProcessError) as error:
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index c742087d1..644923112 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import os
 
 from flexmock import flexmock
 
@@ -7,6 +8,28 @@ from atticmatic.tests.builtins import builtins_mock
 from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
+def test_initialize_with_passphrase_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({'encryption_passphrase': 'pass'}, command='attic')
+        assert os.environ.get('ATTIC_PASSPHRASE') == 'pass'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_without_passphrase_should_not_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({}, command='attic')
+        assert os.environ.get('ATTIC_PASSPHRASE') == None
+    finally:
+        os.environ = orig_environ
+
+
 def insert_subprocess_mock(check_call_command, **kwargs):
     subprocess = flexmock()
     subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
@@ -41,6 +64,7 @@ def test_create_archive_should_call_attic_with_parameters():
     module.create_archive(
         excludes_filename='excludes',
         verbosity=None,
+        storage_config={},
         source_directories='foo bar',
         repository='repo',
         command='attic',
@@ -55,6 +79,7 @@ def test_create_archive_with_none_excludes_filename_should_call_attic_without_ex
     module.create_archive(
         excludes_filename=None,
         verbosity=None,
+        storage_config={},
         source_directories='foo bar',
         repository='repo',
         command='attic',
@@ -69,6 +94,7 @@ def test_create_archive_with_verbosity_some_should_call_attic_with_stats_paramet
     module.create_archive(
         excludes_filename='excludes',
         verbosity=VERBOSITY_SOME,
+        storage_config={},
         source_directories='foo bar',
         repository='repo',
         command='attic',
@@ -83,6 +109,22 @@ def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_param
     module.create_archive(
         excludes_filename='excludes',
         verbosity=VERBOSITY_LOTS,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_compression_should_call_attic_with_compression_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={'compression': 'rle'},
         source_directories='foo bar',
         repository='repo',
         command='attic',
diff --git a/sample/config b/sample/config
index 82d77d12b..cf4b391ab 100644
--- a/sample/config
+++ b/sample/config
@@ -5,6 +5,15 @@ source_directories: /home /etc
 # Path to local or remote repository.
 repository: user@backupserver:sourcehostname.attic
 
+[storage]
+# Passphrase to unlock the encryption key with. Only use on repositories that
+# were initialized with passphrase/repokey encryption.
+#encryption_passphrase: foo
+# For Borg only, you can specify the type of compression to use when creating
+# archives. See https://borgbackup.github.io/borgbackup/usage.html#borg-create
+# for details. Defaults to no compression.
+#compression: lz4
+
 [retention]
 # Retention policy for how many backups to keep in each category. See
 # https://attic-backup.org/usage.html#attic-prune or
diff --git a/setup.py b/setup.py
index ad32a1458..73a779613 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.1.5'
+VERSION = '0.1.6'
 
 
 setup(

From 5c58f85be18be79de0785c025b517dbc88a85f17 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Wed, 2 Sep 2015 22:48:14 -0700
Subject: [PATCH 076/189] Added tag 0.1.6 for changeset 4c63f3d90ec2

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 076d808e4..fed172850 100644
--- a/.hgtags
+++ b/.hgtags
@@ -20,3 +20,4 @@ acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
 6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
 e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 0afff209b902698c2266986129d6dc9f5f913101 0.1.5
+4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6

From 6dc0173b74da512896e40e22009f71fd51cb0df7 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 6 Sep 2015 15:33:56 -0700
Subject: [PATCH 077/189] #11: Fixed parsing of punctuation in configuration
 file.

---
 NEWS                                        |  4 +++
 atticmatic/config.py                        | 10 +++----
 atticmatic/tests/integration/test_config.py | 29 +++++++++++++++++++++
 atticmatic/tests/unit/test_config.py        |  2 +-
 setup.py                                    |  2 +-
 5 files changed, 40 insertions(+), 7 deletions(-)
 create mode 100644 atticmatic/tests/integration/test_config.py

diff --git a/NEWS b/NEWS
index d5185023b..2a7c7aee5 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.1.7-dev
+
+ * #11: Fixed parsing of punctuation in configuration file.
+
 0.1.6
 
  * #9: New configuration option for the encryption passphrase.
diff --git a/atticmatic/config.py b/atticmatic/config.py
index bc1a32d16..4f45b89bd 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -2,10 +2,10 @@ from collections import OrderedDict, namedtuple
 
 try:
     # Python 2
-    from ConfigParser import ConfigParser
+    from ConfigParser import RawConfigParser
 except ImportError:
     # Python 3
-    from configparser import ConfigParser
+    from configparser import RawConfigParser
 
 
 Section_format = namedtuple('Section_format', ('name', 'options'))
@@ -22,7 +22,7 @@ def option(name, value_type=str, required=True):
 
 def validate_configuration_format(parser, config_format):
     '''
-    Given an open ConfigParser and an expected config file format, validate that the parsed
+    Given an open RawConfigParser and an expected config file format, validate that the parsed
     configuration file has the expected sections, that any required options are present in those
     sections, and that there aren't any unexpected options.
 
@@ -83,7 +83,7 @@ def validate_configuration_format(parser, config_format):
 
 def parse_section_options(parser, section_format):
     '''
-    Given an open ConfigParser and an expected section format, return the option values from that
+    Given an open RawConfigParser and an expected section format, return the option values from that
     section as a dict mapping from option name to value. Omit those options that are not present in
     the parsed options.
 
@@ -108,7 +108,7 @@ def parse_configuration(config_filename, config_format):
 
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
-    parser = ConfigParser()
+    parser = RawConfigParser()
     parser.read(config_filename)
 
     validate_configuration_format(parser, config_format)
diff --git a/atticmatic/tests/integration/test_config.py b/atticmatic/tests/integration/test_config.py
new file mode 100644
index 000000000..31fcd9e34
--- /dev/null
+++ b/atticmatic/tests/integration/test_config.py
@@ -0,0 +1,29 @@
+try:
+    # Python 2
+    from cStringIO import StringIO
+except ImportError:
+    # Python 3
+    from io import StringIO
+
+from collections import OrderedDict
+import string
+
+from atticmatic import config as module
+
+
+def test_parse_section_options_with_punctuation_should_return_section_options():
+    parser = module.RawConfigParser()
+    parser.readfp(StringIO('[section]\nfoo: {}\n'.format(string.punctuation)))
+
+    section_format = module.Section_format(
+        'section',
+        (module.Config_option('foo', str, required=True),),
+    )
+
+    config = module.parse_section_options(parser, section_format)
+
+    assert config == OrderedDict(
+        (
+            ('foo', string.punctuation),
+        )
+    )
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index 4c889b091..7a1866e85 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -198,7 +198,7 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict():
 def insert_mock_parser():
     parser = flexmock()
     parser.should_receive('read')
-    module.ConfigParser = lambda: parser
+    module.RawConfigParser = lambda: parser
 
     return parser
 
diff --git a/setup.py b/setup.py
index 73a779613..9dc31c7f1 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.1.6'
+VERSION = '0.1.7-dev'
 
 
 setup(

From 8a58b72934c4f84d667dcce0402e0789f93c3af1 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 6 Sep 2015 15:55:14 -0700
Subject: [PATCH 078/189] Better error message when configuration file is
 missing.

---
 NEWS                                 |  1 +
 atticmatic/config.py                 |  3 ++-
 atticmatic/tests/unit/test_config.py | 10 +++++++++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 2a7c7aee5..1b46c71f3 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,7 @@
 0.1.7-dev
 
  * #11: Fixed parsing of punctuation in configuration file.
+ * Better error message when configuration file is missing.
 
 0.1.6
 
diff --git a/atticmatic/config.py b/atticmatic/config.py
index 4f45b89bd..ddf574d77 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -109,7 +109,8 @@ def parse_configuration(config_filename, config_format):
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
     parser = RawConfigParser()
-    parser.read(config_filename)
+    if not parser.read(config_filename):
+        raise ValueError('Configuration file cannot be opened: {}'.format(config_filename))
 
     validate_configuration_format(parser, config_format)
 
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index 7a1866e85..4b4d3a7c2 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -197,7 +197,7 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict():
 
 def insert_mock_parser():
     parser = flexmock()
-    parser.should_receive('read')
+    parser.should_receive('read').and_return([flexmock()])
     module.RawConfigParser = lambda: parser
 
     return parser
@@ -220,3 +220,11 @@ def test_parse_configuration_should_return_section_configs():
     parsed_config = module.parse_configuration('filename', config_format)
 
     assert parsed_config == type(parsed_config)(*mock_section_configs)
+
+
+def test_parse_configuration_with_file_open_error_should_raise():
+    parser = insert_mock_parser()
+    parser.should_receive('read').and_return([])
+
+    with assert_raises(ValueError):
+        module.parse_configuration('filename', config_format=flexmock())

From 2456fc67f17679204b9a1052c80d5b0cc462f54d Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 6 Sep 2015 16:40:39 -0700
Subject: [PATCH 079/189] Revving version.

---
 NEWS     | 2 +-
 setup.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 1b46c71f3..387a46d98 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
-0.1.7-dev
+0.1.7
 
  * #11: Fixed parsing of punctuation in configuration file.
  * Better error message when configuration file is missing.
diff --git a/setup.py b/setup.py
index 9dc31c7f1..343d69f2c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.1.7-dev'
+VERSION = '0.1.7'
 
 
 setup(

From 944c0212c38690e3120541b3b0a244d23ed1743e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 6 Sep 2015 16:40:46 -0700
Subject: [PATCH 080/189] Added tag 0.1.7 for changeset 5a458ebef804

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index fed172850..072fbd2b2 100644
--- a/.hgtags
+++ b/.hgtags
@@ -21,3 +21,4 @@ acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
 e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 0afff209b902698c2266986129d6dc9f5f913101 0.1.5
 4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6
+5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7

From fa7955b8cf9a989ca35cbd3b34b50dfc6a4eb771 Mon Sep 17 00:00:00 2001
From: TW <tw@waldmann-edv.de>
Date: Tue, 20 Oct 2015 23:08:43 +0200
Subject: [PATCH 081/189] fixed typo in README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3d9d9991a..9c38cb5a0 100644
--- a/README.md
+++ b/README.md
@@ -117,7 +117,7 @@ Then, to actually run tests, run:
 ### Broken pipe with remote repository
 
 When running atticmatic on a large remote repository, you may receive errors
-like the following, particularly while "attic check" is valiating backups for
+like the following, particularly while "attic check" is validating backups for
 consistency:
 
     Write failed: Broken pipe

From 80318e6e30fd90d1dce38e862159cd983450dd3d Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 8 Nov 2015 17:03:40 -0800
Subject: [PATCH 082/189] Removed tag github/yaml_config_files

---
 .hgtags | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.hgtags b/.hgtags
index 072fbd2b2..0e2b56994 100644
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,5 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 0afff209b902698c2266986129d6dc9f5f913101 0.1.5
 4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6
 5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7
+977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files
+0000000000000000000000000000000000000000 github/yaml_config_files

From 3a3851d2a5d6309a4672b7e2d26f91403bdda1eb Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 8 Nov 2015 17:04:14 -0800
Subject: [PATCH 083/189] Removed tag github/yaml_config_files

---
 .hgtags | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.hgtags b/.hgtags
index 072fbd2b2..0e2b56994 100644
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,5 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 0afff209b902698c2266986129d6dc9f5f913101 0.1.5
 4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6
 5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7
+977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files
+0000000000000000000000000000000000000000 github/yaml_config_files

From e59845d4e18b339e49f66cd8c0b9b3828d42e8f7 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 8 Nov 2015 17:06:48 -0800
Subject: [PATCH 084/189] Added tag github/master for changeset 28434dd0440c

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 0e2b56994..b58095904 100644
--- a/.hgtags
+++ b/.hgtags
@@ -24,3 +24,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7
 977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files
 0000000000000000000000000000000000000000 github/yaml_config_files
+28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master

From 32858fb0b4b97fc73f7969439c5207b90c7f97ca Mon Sep 17 00:00:00 2001
From: Robin Schneider <ypid@riseup.net>
Date: Wed, 20 Jan 2016 13:11:15 +0100
Subject: [PATCH 085/189] Also allow the INI example to be highlighted on
 GitHub.

---
 README.md | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 9c38cb5a0..509b1dd20 100644
--- a/README.md
+++ b/README.md
@@ -12,22 +12,24 @@ all on the command-line, and handles common errors.
 
 Here's an example config file:
 
-    [location]
-    # Space-separated list of source directories to backup.
-    source_directories: /home /etc
+```INI
+[location]
+# Space-separated list of source directories to backup.
+source_directories: /home /etc
 
-    # Path to local or remote backup repository.
-    repository: user@backupserver:sourcehostname.attic
+# Path to local or remote backup repository.
+repository: user@backupserver:sourcehostname.attic
 
-    [retention]
-    # Retention policy for how many backups to keep in each category.
-    keep_daily: 7
-    keep_weekly: 4
-    keep_monthly: 6
+[retention]
+# Retention policy for how many backups to keep in each category.
+keep_daily: 7
+keep_weekly: 4
+keep_monthly: 6
 
-    [consistency]
-    # Consistency checks to run, or "disabled" to prevent checks.
-    checks: repository archives
+[consistency]
+# Consistency checks to run, or "disabled" to prevent checks.
+checks: repository archives
+```
 
 Additionally, exclude patterns can be specified in a separate excludes config
 file, one pattern per line.

From 978096b402de91cdb1c1064a84ca0596504d040d Mon Sep 17 00:00:00 2001
From: Robin Schneider <ypid@riseup.net>
Date: Sat, 30 Jan 2016 20:24:29 +0100
Subject: [PATCH 086/189] Added .gitignore file.

---
 .gitignore | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000..2a5300705
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,6 @@
+*.egg-info
+*.pyc
+*.swp
+.tox
+build
+dist

From 9e52be6ffd6d7d1b79b5bb179859ccceba89c7af Mon Sep 17 00:00:00 2001
From: Robin Schneider <ypid@riseup.net>
Date: Sun, 31 Jan 2016 11:42:07 +0100
Subject: [PATCH 087/189] Use /\s+/ to split source_directories to handle 1+
 spaces.

This bug is can be quite annoying because when you accidentally used
something like:

```ini
[location]
source_directories: backup_one  backup_two
;                              A (Additional space here)
```

It would call Attic/Borg with ('backup_one', '', 'backup_two') which in
turn backups your whole $PWD.
---
 NEWS                                          |  4 ++++
 atticmatic/backends/shared.py                 |  5 +++--
 atticmatic/tests/unit/backends/test_shared.py | 15 +++++++++++++++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 387a46d98..b7227b80d 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+0.1.8-dev
+
+ * Fixed handling of repeated spaces in source_directories which resulted in backup up everything.
+
 0.1.7
 
  * #11: Fixed parsing of punctuation in configuration file.
diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index 5baff3a92..a60aa3bbe 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -1,5 +1,6 @@
 from datetime import datetime
 import os
+import re
 import platform
 import subprocess
 
@@ -63,7 +64,7 @@ def create_archive(
     list of source directories, a local or remote repository path, and a command to run, create an
     attic archive.
     '''
-    sources = tuple(source_directories.split(' '))
+    sources = tuple(re.split('\s+', source_directories))
     exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
@@ -167,7 +168,7 @@ def _make_check_flags(checks, check_last=None):
         ('repository',)
 
     This will be returned as:
-    
+
         ('--repository-only',)
 
     Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index 644923112..ddb090fc6 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -71,6 +71,21 @@ def test_create_archive_should_call_attic_with_parameters():
     )
 
 
+def test_create_archive_with_two_spaces_in_source_directories():
+    insert_subprocess_mock(CREATE_COMMAND)
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={},
+        source_directories='foo  bar',
+        repository='repo',
+        command='attic',
+    )
+
+
 def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes():
     insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
     insert_platform_mock()

From 049f9c8853ef287a547e55ab2b3d02ea6b0c6cd7 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 13 Feb 2016 10:43:31 -0800
Subject: [PATCH 088/189] Added support for --one-file-system for Borg.

---
 AUTHORS                                       |  1 +
 NEWS                                          |  3 ++-
 atticmatic/backends/borg.py                   |  9 ++++++++-
 atticmatic/backends/shared.py                 |  5 ++++-
 atticmatic/config.py                          |  1 +
 atticmatic/tests/unit/backends/test_shared.py | 16 ++++++++++++++++
 atticmatic/tests/unit/test_config.py          |  2 ++
 sample/config                                 |  4 ++++
 setup.py                                      |  2 +-
 9 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 2a2f28fea..95d2df819 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -2,3 +2,4 @@ Dan Helfman <witten@torsion.org>: Main developer
 
 Alexander Görtz: Python 3 compatibility
 Henning Schroeder: Copy editing
+Robin `ypid` Schneider: Support additional options of Borg
diff --git a/NEWS b/NEWS
index b7227b80d..1b0963f44 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,7 @@
-0.1.8-dev
+0.1.8.dev0
 
  * Fixed handling of repeated spaces in source_directories which resulted in backup up everything.
+ * Added support for --one-file-system for Borg.
 
 0.1.7
 
diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py
index 222c7028f..6105a0567 100644
--- a/atticmatic/backends/borg.py
+++ b/atticmatic/backends/borg.py
@@ -7,7 +7,14 @@ from atticmatic.backends import shared
 
 COMMAND = 'borg'
 CONFIG_FORMAT = (
-    shared.CONFIG_FORMAT[0],  # location
+    Section_format(
+        'location',
+        (
+            option('source_directories'),
+            option('one_file_system', value_type=bool, required=False),
+            option('repository'),
+        ),
+    ),
     Section_format(
         'storage',
         (
diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index a60aa3bbe..5544b30a0 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -58,6 +58,7 @@ def initialize(storage_config, command):
 
 def create_archive(
     excludes_filename, verbosity, storage_config, source_directories, repository, command,
+    one_file_system=None,
 ):
     '''
     Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
@@ -68,6 +69,7 @@ def create_archive(
     exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
+    one_file_system_flags = ('--one-file-system',) if one_file_system else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
         VERBOSITY_LOTS: ('--verbose', '--stats'),
@@ -80,7 +82,8 @@ def create_archive(
             hostname=platform.node(),
             timestamp=datetime.now().isoformat(),
         ),
-    ) + sources + exclude_flags + compression_flags + verbosity_flags
+    ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
+        verbosity_flags
 
     subprocess.check_call(full_command)
 
diff --git a/atticmatic/config.py b/atticmatic/config.py
index ddf574d77..8f7ae9a10 100644
--- a/atticmatic/config.py
+++ b/atticmatic/config.py
@@ -92,6 +92,7 @@ def parse_section_options(parser, section_format):
     type_getter = {
         str: parser.get,
         int: parser.getint,
+        bool: parser.getboolean,
     }
 
     return OrderedDict(
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index ddb090fc6..660fc5b94 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -146,6 +146,22 @@ def test_create_archive_with_compression_should_call_attic_with_compression_para
     )
 
 
+def test_create_archive_with_one_file_system_should_call_attic_with_one_file_system_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+        one_file_system=True,
+    )
+
+
 BASE_PRUNE_FLAGS = (
     ('--keep-daily', '1'),
     ('--keep-weekly', '2'),
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index 4b4d3a7c2..88569e106 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -154,6 +154,7 @@ def test_parse_section_options_should_return_section_options():
     parser = flexmock()
     parser.should_receive('get').with_args('section', 'foo').and_return('value')
     parser.should_receive('getint').with_args('section', 'bar').and_return(1)
+    parser.should_receive('getboolean').never()
     parser.should_receive('has_option').with_args('section', 'foo').and_return(True)
     parser.should_receive('has_option').with_args('section', 'bar').and_return(True)
 
@@ -179,6 +180,7 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict():
     parser = flexmock()
     parser.should_receive('get').never()
     parser.should_receive('getint').never()
+    parser.should_receive('getboolean').never()
     parser.should_receive('has_option').with_args('section', 'foo').and_return(False)
     parser.should_receive('has_option').with_args('section', 'bar').and_return(False)
 
diff --git a/sample/config b/sample/config
index cf4b391ab..3d9432fd2 100644
--- a/sample/config
+++ b/sample/config
@@ -2,6 +2,10 @@
 # Space-separated list of source directories to backup.
 source_directories: /home /etc
 
+# For Borg only, you can specify to stay in same file system (do not cross
+# mount points).
+#one_file_system: True
+
 # Path to local or remote repository.
 repository: user@backupserver:sourcehostname.attic
 
diff --git a/setup.py b/setup.py
index 343d69f2c..ff5a4bef7 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.1.7'
+VERSION = '0.1.8.dev0'
 
 
 setup(

From 0012e0cdea8d4b6b57c50721ad39e946d95559b5 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 13 Feb 2016 10:59:43 -0800
Subject: [PATCH 089/189] Support borg create --umask. (Merge PR from ypid.)

---
 NEWS                                          |  1 +
 atticmatic/backends/borg.py                   |  2 ++
 atticmatic/backends/shared.py                 |  4 +++-
 atticmatic/tests/unit/backends/test_shared.py | 15 +++++++++++++++
 sample/config                                 |  2 ++
 5 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 1b0963f44..412e8bc5f 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,7 @@
 
  * Fixed handling of repeated spaces in source_directories which resulted in backup up everything.
  * Added support for --one-file-system for Borg.
+ * Support borg create --umask.
 
 0.1.7
 
diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py
index 6105a0567..d99af4a65 100644
--- a/atticmatic/backends/borg.py
+++ b/atticmatic/backends/borg.py
@@ -20,6 +20,7 @@ CONFIG_FORMAT = (
         (
             option('encryption_passphrase', required=False),
             option('compression', required=False),
+            option('umask', required=False),
         ),
     ),
     shared.CONFIG_FORMAT[2],  # retention
@@ -34,6 +35,7 @@ CONFIG_FORMAT = (
 
 
 initialize = partial(shared.initialize, command=COMMAND)
+
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)
 check_archives = partial(shared.check_archives, command=COMMAND)
diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index 5544b30a0..f300d1e67 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -69,6 +69,8 @@ def create_archive(
     exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
+    umask = storage_config.get('umask', None)
+    umask_flags = ('--umask', str(umask)) if umask else ()
     one_file_system_flags = ('--one-file-system',) if one_file_system else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
@@ -83,7 +85,7 @@ def create_archive(
             timestamp=datetime.now().isoformat(),
         ),
     ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
-        verbosity_flags
+        umask_flags + verbosity_flags
 
     subprocess.check_call(full_command)
 
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index 660fc5b94..8babc5837 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -162,6 +162,21 @@ def test_create_archive_with_one_file_system_should_call_attic_with_one_file_sys
     )
 
 
+def test_create_archive_with_umask_should_call_attic_with_umask_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={'umask': 740},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
 BASE_PRUNE_FLAGS = (
     ('--keep-daily', '1'),
     ('--keep-weekly', '2'),
diff --git a/sample/config b/sample/config
index 3d9432fd2..70527e45c 100644
--- a/sample/config
+++ b/sample/config
@@ -17,6 +17,8 @@ repository: user@backupserver:sourcehostname.attic
 # archives. See https://borgbackup.github.io/borgbackup/usage.html#borg-create
 # for details. Defaults to no compression.
 #compression: lz4
+# For Borg only, you can specify the umask to be used for borg create.
+#umask: 0740
 
 [retention]
 # Retention policy for how many backups to keep in each category. See

From 88da0c30392b8faa62d7c46d6265a735e32fbbc0 Mon Sep 17 00:00:00 2001
From: Robin Schneider <ypid@riseup.net>
Date: Mon, 25 Jan 2016 23:52:16 +0100
Subject: [PATCH 090/189] Added support for file globs in source_directories.

source_directories_glob can be used to enable glob support (defaults to
disabled).
---
 README.md                                     |  1 +
 atticmatic/backends/shared.py                 |  7 ++++++-
 atticmatic/tests/unit/backends/test_shared.py | 16 ++++++++++++++++
 sample/config                                 |  1 +
 4 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 509b1dd20..d29687e1e 100644
--- a/README.md
+++ b/README.md
@@ -16,6 +16,7 @@ Here's an example config file:
 [location]
 # Space-separated list of source directories to backup.
 source_directories: /home /etc
+# source_directories_glob: 1
 
 # Path to local or remote backup repository.
 repository: user@backupserver:sourcehostname.attic
diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index f300d1e67..65a39dfae 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -3,6 +3,8 @@ import os
 import re
 import platform
 import subprocess
+from glob import glob
+from itertools import chain
 
 from atticmatic.config import Section_format, option
 from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
@@ -19,6 +21,7 @@ CONFIG_FORMAT = (
         'location',
         (
             option('source_directories'),
+            option('source_directories_glob', int, required=False),
             option('repository'),
         ),
     ),
@@ -58,7 +61,7 @@ def initialize(storage_config, command):
 
 def create_archive(
     excludes_filename, verbosity, storage_config, source_directories, repository, command,
-    one_file_system=None,
+    one_file_system=None, source_directories_glob=None
 ):
     '''
     Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
@@ -66,6 +69,8 @@ def create_archive(
     attic archive.
     '''
     sources = tuple(re.split('\s+', source_directories))
+    if source_directories_glob:
+        sources = tuple(chain.from_iterable([glob(x) for x in sources]))
     exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index 8babc5837..684c6d6f3 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -177,6 +177,22 @@ def test_create_archive_with_umask_should_call_attic_with_umask_parameters():
     )
 
 
+def test_create_archive_with_globs():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'setup.py', 'setup.cfg'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='setup*',
+        repository='repo',
+        command='attic',
+        source_directories_glob=1,
+    )
+
+
 BASE_PRUNE_FLAGS = (
     ('--keep-daily', '1'),
     ('--keep-weekly', '2'),
diff --git a/sample/config b/sample/config
index 70527e45c..c698a91c7 100644
--- a/sample/config
+++ b/sample/config
@@ -1,6 +1,7 @@
 [location]
 # Space-separated list of source directories to backup.
 source_directories: /home /etc
+# source_directories_glob: 1
 
 # For Borg only, you can specify to stay in same file system (do not cross
 # mount points).

From 953d08ba63ee93f0079a92ef1f50f6dd363d423e Mon Sep 17 00:00:00 2001
From: Robin Schneider <ypid@riseup.net>
Date: Sat, 13 Feb 2016 21:05:34 +0100
Subject: [PATCH 091/189] Made globing for source_directories the default.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Don’t remove non existing files/directories from the list and let
attic/borg handle this.
---
 NEWS                                          | 1 +
 README.md                                     | 4 ++--
 atticmatic/backends/shared.py                 | 8 +++-----
 atticmatic/tests/unit/backends/test_shared.py | 1 -
 sample/config                                 | 4 ++--
 5 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/NEWS b/NEWS
index 412e8bc5f..77042d950 100644
--- a/NEWS
+++ b/NEWS
@@ -3,6 +3,7 @@
  * Fixed handling of repeated spaces in source_directories which resulted in backup up everything.
  * Added support for --one-file-system for Borg.
  * Support borg create --umask.
+ * Added support for file globs in source_directories.
 
 0.1.7
 
diff --git a/README.md b/README.md
index d29687e1e..1082ff822 100644
--- a/README.md
+++ b/README.md
@@ -15,8 +15,8 @@ Here's an example config file:
 ```INI
 [location]
 # Space-separated list of source directories to backup.
-source_directories: /home /etc
-# source_directories_glob: 1
+# Globs are expanded.
+source_directories: /home /etc /var/log/syslog*
 
 # Path to local or remote backup repository.
 repository: user@backupserver:sourcehostname.attic
diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index 65a39dfae..8026540bd 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -21,7 +21,6 @@ CONFIG_FORMAT = (
         'location',
         (
             option('source_directories'),
-            option('source_directories_glob', int, required=False),
             option('repository'),
         ),
     ),
@@ -61,16 +60,15 @@ def initialize(storage_config, command):
 
 def create_archive(
     excludes_filename, verbosity, storage_config, source_directories, repository, command,
-    one_file_system=None, source_directories_glob=None
+    one_file_system=None
 ):
     '''
     Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
     list of source directories, a local or remote repository path, and a command to run, create an
     attic archive.
     '''
-    sources = tuple(re.split('\s+', source_directories))
-    if source_directories_glob:
-        sources = tuple(chain.from_iterable([glob(x) for x in sources]))
+    sources = re.split('\s+', source_directories)
+    sources = tuple(chain.from_iterable([glob(x) if glob(x) else [x] for x in sources]))
     exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index 684c6d6f3..20a98f6c1 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -189,7 +189,6 @@ def test_create_archive_with_globs():
         source_directories='setup*',
         repository='repo',
         command='attic',
-        source_directories_glob=1,
     )
 
 
diff --git a/sample/config b/sample/config
index c698a91c7..e68137675 100644
--- a/sample/config
+++ b/sample/config
@@ -1,7 +1,7 @@
 [location]
 # Space-separated list of source directories to backup.
-source_directories: /home /etc
-# source_directories_glob: 1
+# Globs are expanded.
+source_directories: /home /etc /var/log/syslog*
 
 # For Borg only, you can specify to stay in same file system (do not cross
 # mount points).

From cf545ae93a26bfb2ed023c2407f09d6601ee761a Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 13 Feb 2016 16:41:17 -0800
Subject: [PATCH 092/189] Mocking out glob() in test so it doesn't hit the
 filesystem, and simplifying comprehension.

---
 atticmatic/backends/shared.py                 |  2 +-
 atticmatic/tests/unit/backends/test_shared.py | 39 +++++++++++++++++--
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index 8026540bd..417a732a1 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -68,7 +68,7 @@ def create_archive(
     attic archive.
     '''
     sources = re.split('\s+', source_directories)
-    sources = tuple(chain.from_iterable([glob(x) if glob(x) else [x] for x in sources]))
+    sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources))
     exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index 20a98f6c1..cddef00f0 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -177,16 +177,49 @@ def test_create_archive_with_umask_should_call_attic_with_umask_parameters():
     )
 
 
-def test_create_archive_with_globs():
-    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'setup.py', 'setup.cfg'))
+def test_create_archive_with_source_directories_glob_expands():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
+    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
         excludes_filename=None,
         verbosity=None,
         storage_config={},
-        source_directories='setup*',
+        source_directories='foo*',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_non_matching_source_directories_glob_passes_through():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo*'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).should_receive('glob').with_args('foo*').and_return([])
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='foo*',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_glob_should_call_attic_with_expanded_directories():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='foo*',
         repository='repo',
         command='attic',
     )

From 15bf273e6e998e4064a3d7009d404ee660c549a8 Mon Sep 17 00:00:00 2001
From: Jan Gondol <gondol@gondol.sk>
Date: Wed, 6 Apr 2016 14:54:06 +0200
Subject: [PATCH 093/189] Fix broken link to Borg quickstart

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1082ff822..febacf0e2 100644
--- a/README.md
+++ b/README.md
@@ -45,7 +45,7 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on
 
 To get up and running, follow the [Attic Quick
 Start](https://attic-backup.org/quickstart.html) or the [Borg Quick
-Start](https://borgbackup.github.io/borgbackup/quickstart.html) to create a
+Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create a
 repository on a local or remote host. Note that if you plan to run atticmatic
 on a schedule with cron, and you encrypt your attic repository with a
 passphrase instead of a key file, you'll need to set the atticmatic

From c3b4cb21ed77d5477026ec59d91c2ace18ee462e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 10 Apr 2016 10:23:32 -0700
Subject: [PATCH 094/189] Fixed links to Borg documentation.

---
 NEWS          | 1 +
 README.md     | 2 +-
 sample/config | 6 +++---
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/NEWS b/NEWS
index 77042d950..a2e7afa43 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,7 @@
  * Added support for --one-file-system for Borg.
  * Support borg create --umask.
  * Added support for file globs in source_directories.
+ * Fixed links to Borg documentation.
 
 0.1.7
 
diff --git a/README.md b/README.md
index febacf0e2..aef10786b 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ title: Atticmatic
 
 atticmatic is a simple Python wrapper script for the
 [Attic](https://attic-backup.org/) and
-[Borg](https://borgbackup.github.io/borgbackup/) backup software that
+[Borg](https://borgbackup.readthedocs.org/en/stable/) backup software that
 initiates a backup, prunes any old backups according to a retention policy,
 and validates backups for consistency. The script supports specifying your
 settings in a declarative configuration file rather than having to put them
diff --git a/sample/config b/sample/config
index e68137675..5a4c2e2d2 100644
--- a/sample/config
+++ b/sample/config
@@ -15,7 +15,7 @@ repository: user@backupserver:sourcehostname.attic
 # were initialized with passphrase/repokey encryption.
 #encryption_passphrase: foo
 # For Borg only, you can specify the type of compression to use when creating
-# archives. See https://borgbackup.github.io/borgbackup/usage.html#borg-create
+# archives. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
 # for details. Defaults to no compression.
 #compression: lz4
 # For Borg only, you can specify the umask to be used for borg create.
@@ -24,7 +24,7 @@ repository: user@backupserver:sourcehostname.attic
 [retention]
 # Retention policy for how many backups to keep in each category. See
 # https://attic-backup.org/usage.html#attic-prune or
-# https://borgbackup.github.io/borgbackup/usage.html#borg-prune for details.
+# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
 #keep_within: 3H
 #keep_hourly: 24
 keep_daily: 7
@@ -37,7 +37,7 @@ keep_yearly: 1
 # Space-separated list of consistency checks to run: "repository", "archives",
 # or both. Defaults to both. Set to "disabled" to disable all consistency
 # checks. See https://attic-backup.org/usage.html#attic-check or
-# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details.
+# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
 checks: repository archives
 # For Borg only, you can restrict the number of checked archives to the last n.
 #check_last: 3

From fa87aed263b163bb3e79689fce8390379ab9576e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 10 Apr 2016 15:27:21 -0700
Subject: [PATCH 095/189] Normalizing recent changes. No new content.

---
 NEWS | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/NEWS b/NEWS
index a2e7afa43..ddcfe7888 100644
--- a/NEWS
+++ b/NEWS
@@ -1,10 +1,10 @@
 0.1.8.dev0
 
- * Fixed handling of repeated spaces in source_directories which resulted in backup up everything.
- * Added support for --one-file-system for Borg.
- * Support borg create --umask.
- * Added support for file globs in source_directories.
- * Fixed links to Borg documentation.
+ * Fix for handling of spaces in source_directories which resulted in backup up everything.
+ * Fix for broken links to Borg documentation.
+ * Support for Borg --one-file-system.
+ * Support for Borg create --umask.
+ * support for file globs in source_directories.
 
 0.1.7
 

From 82e8dae94899c6305bacafb2213c130f61c36dfa Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 10 Apr 2016 15:29:42 -0700
Subject: [PATCH 096/189] At verbosity zero, suppressing Borg check spew to
 stderr about "Checking segments".

---
 NEWS                          | 3 ++-
 atticmatic/backends/shared.py | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index ddcfe7888..321eb6f1a 100644
--- a/NEWS
+++ b/NEWS
@@ -2,9 +2,10 @@
 
  * Fix for handling of spaces in source_directories which resulted in backup up everything.
  * Fix for broken links to Borg documentation.
+ * At verbosity zero, suppressing Borg check spew to stderr about "Checking segments".
  * Support for Borg --one-file-system.
  * Support for Borg create --umask.
- * support for file globs in source_directories.
+ * Support for file globs in source_directories.
 
 0.1.7
 
diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py
index 417a732a1..2fe6a5c68 100644
--- a/atticmatic/backends/shared.py
+++ b/atticmatic/backends/shared.py
@@ -213,7 +213,7 @@ def check_archives(verbosity, repository, consistency_config, command):
         repository,
     ) + _make_check_flags(checks, check_last) + verbosity_flags
 
-    # The check command spews to stdout even without the verbose flag. Suppress it.
+    # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
     stdout = None if verbosity_flags else open(os.devnull, 'w')
 
-    subprocess.check_call(full_command, stdout=stdout)
+    subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)

From 8210172d7f8317f45705255f1e353d5b9c2e6697 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 10 Apr 2016 15:46:43 -0700
Subject: [PATCH 097/189] Fixing "check" backend tests to support new use of
 stderr=STDOUT.

---
 atticmatic/tests/unit/backends/test_shared.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py
index cddef00f0..26c8b0da8 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/atticmatic/tests/unit/backends/test_shared.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+from subprocess import STDOUT
 import os
 
 from flexmock import flexmock
@@ -31,7 +32,7 @@ def test_initialize_without_passphrase_should_not_set_environment():
 
 
 def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock()
+    subprocess = flexmock(STDOUT=STDOUT)
     subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
     flexmock(module).subprocess = subprocess
 
@@ -353,7 +354,7 @@ def test_check_archives_should_call_attic_with_parameters():
     stdout = flexmock()
     insert_subprocess_mock(
         ('attic', 'check', 'repo'),
-        stdout=stdout,
+        stdout=stdout, stderr=STDOUT,
     )
     insert_platform_mock()
     insert_datetime_mock()
@@ -374,7 +375,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
         ('attic', 'check', 'repo', '--verbose'),
-        stdout=None,
+        stdout=None, stderr=STDOUT,
     )
     insert_platform_mock()
     insert_datetime_mock()
@@ -393,7 +394,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
         ('attic', 'check', 'repo', '--verbose'),
-        stdout=None,
+        stdout=None, stderr=STDOUT,
     )
     insert_platform_mock()
     insert_datetime_mock()

From 5b66dc69a1b262bed928567205188f634c3e9573 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 10 Apr 2016 15:48:10 -0700
Subject: [PATCH 098/189] Refreshing flexmock version in test requirements.

---
 test_requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_requirements.txt b/test_requirements.txt
index 5a34a857b..8969b43ad 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -1,2 +1,2 @@
-flexmock==0.9.7
+flexmock==0.10.2
 nose==1.3.4

From 0ea58244272e74ed4a083686f67f74e570559c8c Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 10 Apr 2016 15:59:36 -0700
Subject: [PATCH 099/189] Switching from the no-longer-maintained nose test
 runner to pytest.

---
 atticmatic/tests/integration/test_command.py |  4 ++--
 atticmatic/tests/unit/test_config.py         | 12 ++++++------
 setup.cfg                                    |  3 ---
 setup.py                                     |  2 +-
 test_requirements.txt                        |  2 +-
 tox.ini                                      |  2 +-
 6 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py
index 346612bf6..b26d8154b 100644
--- a/atticmatic/tests/integration/test_command.py
+++ b/atticmatic/tests/integration/test_command.py
@@ -2,7 +2,7 @@ import os
 import sys
 
 from flexmock import flexmock
-from nose.tools import assert_raises
+import pytest
 
 from atticmatic import command as module
 
@@ -66,7 +66,7 @@ def test_parse_arguments_with_invalid_arguments_exits():
     sys.stderr = sys.stdout
 
     try:
-        with assert_raises(SystemExit):
+        with pytest.raises(SystemExit):
             module.parse_arguments(COMMAND_NAME, '--posix-me-harder')
     finally:
         sys.stderr = original_stderr
diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py
index 88569e106..6422fa58b 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/atticmatic/tests/unit/test_config.py
@@ -1,7 +1,7 @@
 from collections import OrderedDict
 
 from flexmock import flexmock
-from nose.tools import assert_raises
+import pytest
 
 from atticmatic import config as module
 
@@ -61,7 +61,7 @@ def test_validate_configuration_format_with_missing_required_section_should_rais
         ),
     )
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
@@ -96,7 +96,7 @@ def test_validate_configuration_format_with_unknown_section_should_raise():
         module.Section_format('section', options=()),
     )
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
@@ -114,7 +114,7 @@ def test_validate_configuration_format_with_missing_required_option_should_raise
         ),
     )
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
@@ -146,7 +146,7 @@ def test_validate_configuration_format_with_extra_option_should_raise():
         ),
     )
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
@@ -228,5 +228,5 @@ def test_parse_configuration_with_file_open_error_should_raise():
     parser = insert_mock_parser()
     parser.should_receive('read').and_return([])
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.parse_configuration('filename', config_format=flexmock())
diff --git a/setup.cfg b/setup.cfg
index 222a3da98..12871ff0f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,2 @@
 [metadata]
 description-file=README.md
-
-[nosetests]
-detailed-errors=1
diff --git a/setup.py b/setup.py
index ff5a4bef7..f399973c8 100644
--- a/setup.py
+++ b/setup.py
@@ -30,6 +30,6 @@ setup(
     },
     tests_require=(
         'flexmock',
-        'nose',
+        'pytest',
     )
 )
diff --git a/test_requirements.txt b/test_requirements.txt
index 8969b43ad..3c5f9e5c9 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -1,2 +1,2 @@
 flexmock==0.10.2
-nose==1.3.4
+pytest==2.9.1
diff --git a/tox.ini b/tox.ini
index 6ac577bbf..64d1f6c1f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,4 +5,4 @@ skipsdist=True
 [testenv]
 usedevelop=True
 deps=-rtest_requirements.txt
-commands = nosetests []
+commands = py.test []

From 9e45da75cb642a1df4a4a381070c63931851fa53 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 10 Apr 2016 16:01:05 -0700
Subject: [PATCH 100/189] Cutting a release.

---
 NEWS     | 4 ++--
 setup.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index 321eb6f1a..8e26a4284 100644
--- a/NEWS
+++ b/NEWS
@@ -1,8 +1,8 @@
-0.1.8.dev0
+0.1.8
 
  * Fix for handling of spaces in source_directories which resulted in backup up everything.
  * Fix for broken links to Borg documentation.
- * At verbosity zero, suppressing Borg check spew to stderr about "Checking segments".
+ * At verbosity zero, suppressing Borg check stderr spew about "Checking segments".
  * Support for Borg --one-file-system.
  * Support for Borg create --umask.
  * Support for file globs in source_directories.
diff --git a/setup.py b/setup.py
index f399973c8..9acda9ea2 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.1.8.dev0'
+VERSION = '0.1.8'
 
 
 setup(

From c7e23fe9edb71fa93762db0190a67af19d552498 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 10 Apr 2016 16:01:18 -0700
Subject: [PATCH 101/189] Added tag 0.1.8 for changeset dbc96d3f83bd

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index b58095904..65c26a7e1 100644
--- a/.hgtags
+++ b/.hgtags
@@ -25,3 +25,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files
 0000000000000000000000000000000000000000 github/yaml_config_files
 28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master
+dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8

From 175761c7575ccef848f0638a1e5b8b816c9961a9 Mon Sep 17 00:00:00 2001
From: dawez <dawez1@gmail.com>
Date: Sun, 17 Apr 2016 22:26:07 +0200
Subject: [PATCH 102/189] fixed typos in readme

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index aef10786b..59bf40c96 100644
--- a/README.md
+++ b/README.md
@@ -93,15 +93,15 @@ By default, the backup will proceed silently except in the case of errors. But
 if you'd like to to get additional information about the progress of the
 backup as it proceeds, use the verbosity option:
 
-    atticmattic --verbosity 1
+    atticmatic --verbosity 1
 
 Or, for even more progress spew:
 
-    atticmattic --verbosity 2
+    atticmatic --verbosity 2
 
 If you'd like to see the available command-line arguments, view the help:
 
-    atticmattic --help
+    atticmatic --help
 
 
 ## Running tests

From 633700c0af85eb41ec143f831fed0678abae2d37 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 10 Jun 2016 11:21:53 -0700
Subject: [PATCH 103/189] Dropping support for Attic.

---
 NEWS                                          |  6 ++
 README.md                                     | 61 ++++++---------
 atticmatic/backends/attic.py                  | 14 ----
 atticmatic/backends/borg.py                   | 41 ----------
 atticmatic/command.py                         | 75 -------------------
 atticmatic/tests/unit/__init__.py             |  0
 atticmatic/tests/unit/backends/__init__.py    |  0
 atticmatic/tests/unit/test_command.py         | 33 --------
 {atticmatic => borgmatic}/__init__.py         |  0
 .../backends/shared.py => borgmatic/borg.py   | 50 ++-----------
 borgmatic/command.py                          | 57 ++++++++++++++
 {atticmatic => borgmatic}/config.py           | 39 ++++++++++
 .../backends => borgmatic/tests}/__init__.py  |  0
 {atticmatic => borgmatic}/tests/builtins.py   |  0
 .../tests/integration}/__init__.py            |  0
 .../tests/integration/test_command.py         | 29 ++++---
 .../tests/integration/test_config.py          |  2 +-
 .../tests/integration/test_version.py         |  0
 .../tests/unit}/__init__.py                   |  0
 .../tests/unit/test_borg.py                   |  6 +-
 .../tests/unit/test_config.py                 |  2 +-
 {atticmatic => borgmatic}/verbosity.py        |  0
 sample/atticmatic.cron                        |  3 -
 sample/config                                 | 15 ++--
 setup.py                                      | 16 ++--
 25 files changed, 167 insertions(+), 282 deletions(-)
 delete mode 100644 atticmatic/backends/attic.py
 delete mode 100644 atticmatic/backends/borg.py
 delete mode 100644 atticmatic/command.py
 delete mode 100644 atticmatic/tests/unit/__init__.py
 delete mode 100644 atticmatic/tests/unit/backends/__init__.py
 delete mode 100644 atticmatic/tests/unit/test_command.py
 rename {atticmatic => borgmatic}/__init__.py (100%)
 rename atticmatic/backends/shared.py => borgmatic/borg.py (80%)
 create mode 100644 borgmatic/command.py
 rename {atticmatic => borgmatic}/config.py (80%)
 rename {atticmatic/backends => borgmatic/tests}/__init__.py (100%)
 rename {atticmatic => borgmatic}/tests/builtins.py (100%)
 rename {atticmatic/tests => borgmatic/tests/integration}/__init__.py (100%)
 rename {atticmatic => borgmatic}/tests/integration/test_command.py (72%)
 rename {atticmatic => borgmatic}/tests/integration/test_config.py (94%)
 rename {atticmatic => borgmatic}/tests/integration/test_version.py (100%)
 rename {atticmatic/tests/integration => borgmatic/tests/unit}/__init__.py (100%)
 rename atticmatic/tests/unit/backends/test_shared.py => borgmatic/tests/unit/test_borg.py (98%)
 rename {atticmatic => borgmatic}/tests/unit/test_config.py (99%)
 rename {atticmatic => borgmatic}/verbosity.py (100%)
 delete mode 100644 sample/atticmatic.cron

diff --git a/NEWS b/NEWS
index 8e26a4284..312fc1f7d 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,9 @@
+1.0.0
+
+ * Attic is no longer supported, as there hasn't been any recent development on it.
+   This will allow faster iteration on Borg-specific configuration.
+ * Project renamed from atticmatic to borgmatic.
+
 0.1.8
 
  * Fix for handling of spaces in source_directories which resulted in backup up everything.
diff --git a/README.md b/README.md
index 59bf40c96..71a93001f 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,8 @@
-title: Atticmatic
+title: Borgmatic
 
 ## Overview
 
-atticmatic is a simple Python wrapper script for the
-[Attic](https://attic-backup.org/) and
+borgmatic (formerly atticmatic) is a simple Python wrapper script for the
 [Borg](https://borgbackup.readthedocs.org/en/stable/) backup software that
 initiates a backup, prunes any old backups according to a retention policy,
 and validates backups for consistency. The script supports specifying your
@@ -19,7 +18,7 @@ Here's an example config file:
 source_directories: /home /etc /var/log/syslog*
 
 # Path to local or remote backup repository.
-repository: user@backupserver:sourcehostname.attic
+repository: user@backupserver:sourcehostname.borg
 
 [retention]
 # Retention policy for how many backups to keep in each category.
@@ -35,37 +34,30 @@ checks: repository archives
 Additionally, exclude patterns can be specified in a separate excludes config
 file, one pattern per line.
 
-atticmatic is hosted at <https://torsion.org/atticmatic> with [source code
-available](https://torsion.org/hg/atticmatic). It's also mirrored on
-[GitHub](https://github.com/witten/atticmatic) and
-[BitBucket](https://bitbucket.org/dhelfman/atticmatic) for convenience.
+borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
+available](https://torsion.org/hg/borgmatic). It's also mirrored on
+[GitHub](https://github.com/witten/borgmatic) and
+[BitBucket](https://bitbucket.org/dhelfman/borgmatic) for convenience.
 
 
 ## Setup
 
-To get up and running, follow the [Attic Quick
-Start](https://attic-backup.org/quickstart.html) or the [Borg Quick
-Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create a
-repository on a local or remote host. Note that if you plan to run atticmatic
-on a schedule with cron, and you encrypt your attic repository with a
-passphrase instead of a key file, you'll need to set the atticmatic
+To get up and running, follow the [Borg Quick
+Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
+a repository on a local or remote host. Note that if you plan to run
+borgmatic on a schedule with cron, and you encrypt your Borg repository with
+a passphrase instead of a key file, you'll need to set the borgmatic
 `encryption_passphrase` configuration variable. See the repository encryption
 section of the Quick Start for more info.
 
 If the repository is on a remote host, make sure that your local root user has
 key-based ssh access to the desired user account on the remote host.
 
-To install atticmatic, run the following command to download and install it:
+To install borgmatic, run the following command to download and install it:
 
-    sudo pip install --upgrade atticmatic
+    sudo pip install --upgrade borgmatic
 
-If you are using Attic, copy the following configuration files:
-
-    sudo cp sample/atticmatic.cron /etc/cron.d/atticmatic
-    sudo mkdir /etc/atticmatic/
-    sudo cp sample/config sample/excludes /etc/atticmatic/
-
-If you are using Borg, copy the files like this instead:
+Then, copy the following configuration files:
 
     sudo cp sample/borgmatic.cron /etc/cron.d/borgmatic
     sudo mkdir /etc/borgmatic/
@@ -76,14 +68,9 @@ Lastly, modify the /etc files with your desired configuration.
 
 ## Usage
 
-You can run atticmatic and start a backup simply by invoking it without
+You can run borgmatic and start a backup simply by invoking it without
 arguments:
 
-    atticmatic
-
-Or, if you're using Borg, use this command instead to make use of the Borg
-backend:
-
     borgmatic
 
 This will also prune any old backups as per the configured retention policy,
@@ -93,15 +80,15 @@ By default, the backup will proceed silently except in the case of errors. But
 if you'd like to to get additional information about the progress of the
 backup as it proceeds, use the verbosity option:
 
-    atticmatic --verbosity 1
+    borgmatic --verbosity 1
 
 Or, for even more progress spew:
 
-    atticmatic --verbosity 2
+    borgmatic --verbosity 2
 
 If you'd like to see the available command-line arguments, view the help:
 
-    atticmatic --help
+    borgmatic --help
 
 
 ## Running tests
@@ -119,12 +106,12 @@ Then, to actually run tests, run:
 
 ### Broken pipe with remote repository
 
-When running atticmatic on a large remote repository, you may receive errors
-like the following, particularly while "attic check" is validating backups for
+When running borgmatic on a large remote repository, you may receive errors
+like the following, particularly while "borg check" is validating backups for
 consistency:
 
     Write failed: Broken pipe
-    attic: Error: Connection closed by remote host
+    borg: Error: Connection closed by remote host
 
 This error can be caused by an ssh timeout, which you can rectify by adding
 the following to the ~/.ssh/config file on the client:
@@ -138,8 +125,8 @@ backups.
 
 ## Issues and feedback
 
-Got an issue or an idea for a feature enhancement? Check out the [atticmatic
-issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). In
+Got an issue or an idea for a feature enhancement? Check out the [borgmatic
+issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues). In
 order to create a new issue or comment on an issue, you'll need to [login
 first](https://tree.taiga.io/login).
 
diff --git a/atticmatic/backends/attic.py b/atticmatic/backends/attic.py
deleted file mode 100644
index daa0b2967..000000000
--- a/atticmatic/backends/attic.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from functools import partial
-
-from atticmatic.backends import shared
-
-# An atticmatic backend that supports Attic for actually handling backups.
-
-COMMAND = 'attic'
-CONFIG_FORMAT = shared.CONFIG_FORMAT
-
-
-initialize = partial(shared.initialize, command=COMMAND)
-create_archive = partial(shared.create_archive, command=COMMAND)
-prune_archives = partial(shared.prune_archives, command=COMMAND)
-check_archives = partial(shared.check_archives, command=COMMAND)
diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py
deleted file mode 100644
index d99af4a65..000000000
--- a/atticmatic/backends/borg.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from functools import partial
-
-from atticmatic.config import Section_format, option
-from atticmatic.backends import shared
-
-# An atticmatic backend that supports Borg for actually handling backups.
-
-COMMAND = 'borg'
-CONFIG_FORMAT = (
-    Section_format(
-        'location',
-        (
-            option('source_directories'),
-            option('one_file_system', value_type=bool, required=False),
-            option('repository'),
-        ),
-    ),
-    Section_format(
-        'storage',
-        (
-            option('encryption_passphrase', required=False),
-            option('compression', required=False),
-            option('umask', required=False),
-        ),
-    ),
-    shared.CONFIG_FORMAT[2],  # retention
-    Section_format(
-        'consistency',
-        (
-            option('checks', required=False),
-            option('check_last', required=False),
-        ),
-    )
-)
-
-
-initialize = partial(shared.initialize, command=COMMAND)
-
-create_archive = partial(shared.create_archive, command=COMMAND)
-prune_archives = partial(shared.prune_archives, command=COMMAND)
-check_archives = partial(shared.check_archives, command=COMMAND)
diff --git a/atticmatic/command.py b/atticmatic/command.py
deleted file mode 100644
index 08ea49e8d..000000000
--- a/atticmatic/command.py
+++ /dev/null
@@ -1,75 +0,0 @@
-from __future__ import print_function
-from argparse import ArgumentParser
-from importlib import import_module
-import os
-from subprocess import CalledProcessError
-import sys
-
-from atticmatic.config import parse_configuration
-
-
-DEFAULT_CONFIG_FILENAME_PATTERN = '/etc/{}/config'
-DEFAULT_EXCLUDES_FILENAME_PATTERN = '/etc/{}/excludes'
-
-
-def parse_arguments(command_name, *arguments):
-    '''
-    Given the name of the command with which this script was invoked and command-line arguments,
-    parse the arguments and return them as an ArgumentParser instance. Use the command name to
-    determine the default configuration and excludes paths.
-    '''
-    config_filename_default = DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name)
-    excludes_filename_default = DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name)
-
-    parser = ArgumentParser()
-    parser.add_argument(
-        '-c', '--config',
-        dest='config_filename',
-        default=config_filename_default,
-        help='Configuration filename',
-    )
-    parser.add_argument(
-        '--excludes',
-        dest='excludes_filename',
-        default=excludes_filename_default if os.path.exists(excludes_filename_default) else None,
-        help='Excludes filename',
-    )
-    parser.add_argument(
-        '-v', '--verbosity',
-        type=int,
-        help='Display verbose progress (1 for some, 2 for lots)',
-    )
-
-    return parser.parse_args(arguments)
-
-
-def load_backend(command_name):
-    '''
-    Given the name of the command with which this script was invoked, return the corresponding
-    backend module responsible for actually dealing with backups.
-    '''
-    backend_name = {
-        'atticmatic': 'attic',
-        'borgmatic': 'borg',
-    }.get(command_name, 'attic')
-
-    return import_module('atticmatic.backends.{}'.format(backend_name))
-
-
-def main():
-    try:
-        command_name = os.path.basename(sys.argv[0])
-        args = parse_arguments(command_name, *sys.argv[1:])
-        backend = load_backend(command_name)
-        config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT)
-        repository = config.location['repository']
-
-        backend.initialize(config.storage)
-        backend.create_archive(
-            args.excludes_filename, args.verbosity, config.storage, **config.location
-        )
-        backend.prune_archives(args.verbosity, repository, config.retention)
-        backend.check_archives(args.verbosity, repository, config.consistency)
-    except (ValueError, IOError, CalledProcessError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)
diff --git a/atticmatic/tests/unit/__init__.py b/atticmatic/tests/unit/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/atticmatic/tests/unit/backends/__init__.py b/atticmatic/tests/unit/backends/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/atticmatic/tests/unit/test_command.py b/atticmatic/tests/unit/test_command.py
deleted file mode 100644
index 6a3cdb163..000000000
--- a/atticmatic/tests/unit/test_command.py
+++ /dev/null
@@ -1,33 +0,0 @@
-from flexmock import flexmock
-
-from atticmatic import command as module
-
-
-def test_load_backend_with_atticmatic_command_should_return_attic_backend():
-    backend = flexmock()
-    (
-        flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic')
-        .and_return(backend).once()
-    )
-
-    assert module.load_backend('atticmatic') == backend
-
-
-def test_load_backend_with_unknown_command_should_return_attic_backend():
-    backend = flexmock()
-    (
-        flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic')
-        .and_return(backend).once()
-    )
-
-    assert module.load_backend('unknownmatic') == backend
-
-
-def test_load_backend_with_borgmatic_command_should_return_borg_backend():
-    backend = flexmock()
-    (
-        flexmock(module).should_receive('import_module').with_args('atticmatic.backends.borg')
-        .and_return(backend).once()
-    )
-
-    assert module.load_backend('borgmatic') == backend
diff --git a/atticmatic/__init__.py b/borgmatic/__init__.py
similarity index 100%
rename from atticmatic/__init__.py
rename to borgmatic/__init__.py
diff --git a/atticmatic/backends/shared.py b/borgmatic/borg.py
similarity index 80%
rename from atticmatic/backends/shared.py
rename to borgmatic/borg.py
index 2fe6a5c68..adde7cfb6 100644
--- a/atticmatic/backends/shared.py
+++ b/borgmatic/borg.py
@@ -6,52 +6,16 @@ import subprocess
 from glob import glob
 from itertools import chain
 
-from atticmatic.config import Section_format, option
-from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
-# Common backend functionality shared by Attic and Borg. As the two backup
-# commands diverge, these shared functions will likely need to be replaced
-# with non-shared functions within atticmatic.backends.attic and
-# atticmatic.backends.borg.
+# Integration with Borg for actually handling backups.
 
 
-CONFIG_FORMAT = (
-    Section_format(
-        'location',
-        (
-            option('source_directories'),
-            option('repository'),
-        ),
-    ),
-    Section_format(
-        'storage',
-        (
-            option('encryption_passphrase', required=False),
-        ),
-    ),
-    Section_format(
-        'retention',
-        (
-            option('keep_within', required=False),
-            option('keep_hourly', int, required=False),
-            option('keep_daily', int, required=False),
-            option('keep_weekly', int, required=False),
-            option('keep_monthly', int, required=False),
-            option('keep_yearly', int, required=False),
-            option('prefix', required=False),
-        ),
-    ),
-    Section_format(
-        'consistency',
-        (
-            option('checks', required=False),
-        ),
-    )
-)
+COMMAND = 'borg'
 
 
-def initialize(storage_config, command):
+def initialize(storage_config, command=COMMAND):
     passphrase = storage_config.get('encryption_passphrase')
 
     if passphrase:
@@ -59,7 +23,7 @@ def initialize(storage_config, command):
 
 
 def create_archive(
-    excludes_filename, verbosity, storage_config, source_directories, repository, command,
+    excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND,
     one_file_system=None
 ):
     '''
@@ -115,7 +79,7 @@ def _make_prune_flags(retention_config):
     )
 
 
-def prune_archives(verbosity, repository, retention_config, command):
+def prune_archives(verbosity, repository, retention_config, command=COMMAND):
     '''
     Given a verbosity flag, a local or remote repository path, a retention config dict, and a
     command to run, prune attic archives according the the retention policy specified in that
@@ -191,7 +155,7 @@ def _make_check_flags(checks, check_last=None):
     ) + last_flag
 
 
-def check_archives(verbosity, repository, consistency_config, command):
+def check_archives(verbosity, repository, consistency_config, command=COMMAND):
     '''
     Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
     command to run, check the contained attic archives for consistency.
diff --git a/borgmatic/command.py b/borgmatic/command.py
new file mode 100644
index 000000000..337fc3f5e
--- /dev/null
+++ b/borgmatic/command.py
@@ -0,0 +1,57 @@
+from __future__ import print_function
+from argparse import ArgumentParser
+import os
+from subprocess import CalledProcessError
+import sys
+
+from borgmatic import borg
+from borgmatic.config import parse_configuration, CONFIG_FORMAT
+
+
+DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'
+DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
+
+
+def parse_arguments(*arguments):
+    '''
+    Given the name of the command with which this script was invoked and command-line arguments,
+    parse the arguments and return them as an ArgumentParser instance. Use the command name to
+    determine the default configuration and excludes paths.
+    '''
+    parser = ArgumentParser()
+    parser.add_argument(
+        '-c', '--config',
+        dest='config_filename',
+        default=DEFAULT_CONFIG_FILENAME,
+        help='Configuration filename',
+    )
+    parser.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        default=DEFAULT_EXCLUDES_FILENAME if os.path.exists(DEFAULT_EXCLUDES_FILENAME) else None,
+        help='Excludes filename',
+    )
+    parser.add_argument(
+        '-v', '--verbosity',
+        type=int,
+        help='Display verbose progress (1 for some, 2 for lots)',
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():
+    try:
+        args = parse_arguments(*sys.argv[1:])
+        config = parse_configuration(args.config_filename, CONFIG_FORMAT)
+        repository = config.location['repository']
+
+        borg.initialize(config.storage)
+        borg.create_archive(
+            args.excludes_filename, args.verbosity, config.storage, **config.location
+        )
+        borg.prune_archives(args.verbosity, repository, config.retention)
+        borg.check_archives(args.verbosity, repository, config.consistency)
+    except (ValueError, IOError, CalledProcessError) as error:
+        print(error, file=sys.stderr)
+        sys.exit(1)
diff --git a/atticmatic/config.py b/borgmatic/config.py
similarity index 80%
rename from atticmatic/config.py
rename to borgmatic/config.py
index 8f7ae9a10..d7a9a0849 100644
--- a/atticmatic/config.py
+++ b/borgmatic/config.py
@@ -20,6 +20,45 @@ def option(name, value_type=str, required=True):
     return Config_option(name, value_type, required)
 
 
+CONFIG_FORMAT = (
+    Section_format(
+        'location',
+        (
+            option('source_directories'),
+            option('one_file_system', value_type=bool, required=False),
+            option('repository'),
+        ),
+    ),
+    Section_format(
+        'storage',
+        (
+            option('encryption_passphrase', required=False),
+            option('compression', required=False),
+            option('umask', required=False),
+        ),
+    ),
+    Section_format(
+        'retention',
+        (
+            option('keep_within', required=False),
+            option('keep_hourly', int, required=False),
+            option('keep_daily', int, required=False),
+            option('keep_weekly', int, required=False),
+            option('keep_monthly', int, required=False),
+            option('keep_yearly', int, required=False),
+            option('prefix', required=False),
+        ),
+    ),
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+            option('check_last', required=False),
+        ),
+    )
+)
+
+
 def validate_configuration_format(parser, config_format):
     '''
     Given an open RawConfigParser and an expected config file format, validate that the parsed
diff --git a/atticmatic/backends/__init__.py b/borgmatic/tests/__init__.py
similarity index 100%
rename from atticmatic/backends/__init__.py
rename to borgmatic/tests/__init__.py
diff --git a/atticmatic/tests/builtins.py b/borgmatic/tests/builtins.py
similarity index 100%
rename from atticmatic/tests/builtins.py
rename to borgmatic/tests/builtins.py
diff --git a/atticmatic/tests/__init__.py b/borgmatic/tests/integration/__init__.py
similarity index 100%
rename from atticmatic/tests/__init__.py
rename to borgmatic/tests/integration/__init__.py
diff --git a/atticmatic/tests/integration/test_command.py b/borgmatic/tests/integration/test_command.py
similarity index 72%
rename from atticmatic/tests/integration/test_command.py
rename to borgmatic/tests/integration/test_command.py
index b26d8154b..c1e311c59 100644
--- a/atticmatic/tests/integration/test_command.py
+++ b/borgmatic/tests/integration/test_command.py
@@ -4,26 +4,23 @@ import sys
 from flexmock import flexmock
 import pytest
 
-from atticmatic import command as module
-
-
-COMMAND_NAME = 'foomatic'
+from borgmatic import command as module
 
 
 def test_parse_arguments_with_no_arguments_uses_defaults():
     flexmock(os.path).should_receive('exists').and_return(True)
 
-    parser = module.parse_arguments(COMMAND_NAME)
+    parser = module.parse_arguments()
 
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
-    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
     assert parser.verbosity == None
 
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
     flexmock(os.path).should_receive('exists').and_return(True)
 
-    parser = module.parse_arguments(COMMAND_NAME, '--config', 'myconfig', '--excludes', 'myexcludes')
+    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
 
     assert parser.config_filename == 'myconfig'
     assert parser.excludes_filename == 'myexcludes'
@@ -33,9 +30,9 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
 def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
     flexmock(os.path).should_receive('exists').and_return(False)
 
-    parser = module.parse_arguments(COMMAND_NAME)
+    parser = module.parse_arguments()
 
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
     assert parser.excludes_filename == None
     assert parser.verbosity == None
 
@@ -43,9 +40,9 @@ def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_non
 def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
     flexmock(os.path).should_receive('exists').and_return(False)
 
-    parser = module.parse_arguments(COMMAND_NAME, '--excludes', 'myexcludes')
+    parser = module.parse_arguments('--excludes', 'myexcludes')
 
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
     assert parser.excludes_filename == 'myexcludes'
     assert parser.verbosity == None
 
@@ -53,10 +50,10 @@ def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(
 def test_parse_arguments_with_verbosity_flag_overrides_default():
     flexmock(os.path).should_receive('exists').and_return(True)
 
-    parser = module.parse_arguments(COMMAND_NAME, '--verbosity', '1')
+    parser = module.parse_arguments('--verbosity', '1')
 
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
-    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
     assert parser.verbosity == 1
 
 
@@ -67,6 +64,6 @@ def test_parse_arguments_with_invalid_arguments_exits():
 
     try:
         with pytest.raises(SystemExit):
-            module.parse_arguments(COMMAND_NAME, '--posix-me-harder')
+            module.parse_arguments('--posix-me-harder')
     finally:
         sys.stderr = original_stderr
diff --git a/atticmatic/tests/integration/test_config.py b/borgmatic/tests/integration/test_config.py
similarity index 94%
rename from atticmatic/tests/integration/test_config.py
rename to borgmatic/tests/integration/test_config.py
index 31fcd9e34..e849b4f76 100644
--- a/atticmatic/tests/integration/test_config.py
+++ b/borgmatic/tests/integration/test_config.py
@@ -8,7 +8,7 @@ except ImportError:
 from collections import OrderedDict
 import string
 
-from atticmatic import config as module
+from borgmatic import config as module
 
 
 def test_parse_section_options_with_punctuation_should_return_section_options():
diff --git a/atticmatic/tests/integration/test_version.py b/borgmatic/tests/integration/test_version.py
similarity index 100%
rename from atticmatic/tests/integration/test_version.py
rename to borgmatic/tests/integration/test_version.py
diff --git a/atticmatic/tests/integration/__init__.py b/borgmatic/tests/unit/__init__.py
similarity index 100%
rename from atticmatic/tests/integration/__init__.py
rename to borgmatic/tests/unit/__init__.py
diff --git a/atticmatic/tests/unit/backends/test_shared.py b/borgmatic/tests/unit/test_borg.py
similarity index 98%
rename from atticmatic/tests/unit/backends/test_shared.py
rename to borgmatic/tests/unit/test_borg.py
index 26c8b0da8..1c032bbb0 100644
--- a/atticmatic/tests/unit/backends/test_shared.py
+++ b/borgmatic/tests/unit/test_borg.py
@@ -4,9 +4,9 @@ import os
 
 from flexmock import flexmock
 
-from atticmatic.backends import shared as module
-from atticmatic.tests.builtins import builtins_mock
-from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+from borgmatic import borg as module
+from borgmatic.tests.builtins import builtins_mock
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
 def test_initialize_with_passphrase_should_set_environment():
diff --git a/atticmatic/tests/unit/test_config.py b/borgmatic/tests/unit/test_config.py
similarity index 99%
rename from atticmatic/tests/unit/test_config.py
rename to borgmatic/tests/unit/test_config.py
index 6422fa58b..01e21bb0f 100644
--- a/atticmatic/tests/unit/test_config.py
+++ b/borgmatic/tests/unit/test_config.py
@@ -3,7 +3,7 @@ from collections import OrderedDict
 from flexmock import flexmock
 import pytest
 
-from atticmatic import config as module
+from borgmatic import config as module
 
 
 def test_option_should_create_config_option():
diff --git a/atticmatic/verbosity.py b/borgmatic/verbosity.py
similarity index 100%
rename from atticmatic/verbosity.py
rename to borgmatic/verbosity.py
diff --git a/sample/atticmatic.cron b/sample/atticmatic.cron
deleted file mode 100644
index 39bc6bcf0..000000000
--- a/sample/atticmatic.cron
+++ /dev/null
@@ -1,3 +0,0 @@
-# You can drop this file into /etc/cron.d/ to run atticmatic nightly.
-
-0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/atticmatic
diff --git a/sample/config b/sample/config
index 5a4c2e2d2..acea8d7cc 100644
--- a/sample/config
+++ b/sample/config
@@ -8,22 +8,21 @@ source_directories: /home /etc /var/log/syslog*
 #one_file_system: True
 
 # Path to local or remote repository.
-repository: user@backupserver:sourcehostname.attic
+repository: user@backupserver:sourcehostname.borg
 
 [storage]
 # Passphrase to unlock the encryption key with. Only use on repositories that
 # were initialized with passphrase/repokey encryption.
 #encryption_passphrase: foo
-# For Borg only, you can specify the type of compression to use when creating
-# archives. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
+# Type of compression to use when creating archives. See
+# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
 # for details. Defaults to no compression.
 #compression: lz4
-# For Borg only, you can specify the umask to be used for borg create.
+# Umask to be used for borg create.
 #umask: 0740
 
 [retention]
 # Retention policy for how many backups to keep in each category. See
-# https://attic-backup.org/usage.html#attic-prune or
 # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
 #keep_within: 3H
 #keep_hourly: 24
@@ -36,8 +35,8 @@ keep_yearly: 1
 [consistency]
 # Space-separated list of consistency checks to run: "repository", "archives",
 # or both. Defaults to both. Set to "disabled" to disable all consistency
-# checks. See https://attic-backup.org/usage.html#attic-check or
-# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
+# checks. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
+# for details.
 checks: repository archives
-# For Borg only, you can restrict the number of checked archives to the last n.
+# Restrict the number of checked archives to the last n.
 #check_last: 3
diff --git a/setup.py b/setup.py
index 9acda9ea2..f9339766c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,17 +1,17 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.1.8'
+VERSION = '1.0.0'
 
 
 setup(
-    name='atticmatic',
+    name='borgmatic',
     version=VERSION,
-    description='A wrapper script for Attic/Borg backup software that creates and prunes backups',
+    description='A wrapper script for Borg backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',
-    url='https://torsion.org/atticmatic',
-    download_url='https://torsion.org/hg/atticmatic/archive/%s.tar.gz' % VERSION,
+    url='https://torsion.org/borgmatic',
+    download_url='https://torsion.org/hg/borgmatic/archive/%s.tar.gz' % VERSION,
     classifiers=(
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
@@ -24,10 +24,12 @@ setup(
     packages=find_packages(),
     entry_points={
         'console_scripts': [
-            'atticmatic = atticmatic.command:main',
-            'borgmatic = atticmatic.command:main',
+            'borgmatic = borgmatic.command:main',
         ]
     },
+    obsoletes=(
+        'atticmatic',
+    ),
     tests_require=(
         'flexmock',
         'pytest',

From 4533fec167eeb4c9a0560c24ea5909834920604d Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 10 Jun 2016 11:53:45 -0700
Subject: [PATCH 104/189] Documenting how to upgrade from atticmatic to
 borgmatic.

---
 NEWS      |  8 +++++---
 README.md | 20 ++++++++++++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index 312fc1f7d..7b7dccb6f 100644
--- a/NEWS
+++ b/NEWS
@@ -1,8 +1,10 @@
 1.0.0
 
- * Attic is no longer supported, as there hasn't been any recent development on it.
-   This will allow faster iteration on Borg-specific configuration.
- * Project renamed from atticmatic to borgmatic.
+ * Attic is no longer supported, as there hasn't been any recent development on
+   it. Dropping Attic support will allow faster iteration on Borg-specific
+   features. If you're still using Attic, this is a good time to switch to Borg!
+ * Project renamed from atticmatic to borgmatic. See the borgmatic README for
+   information on upgrading.
 
 0.1.8
 
diff --git a/README.md b/README.md
index 71a93001f..b9e1d9348 100644
--- a/README.md
+++ b/README.md
@@ -66,6 +66,26 @@ Then, copy the following configuration files:
 Lastly, modify the /etc files with your desired configuration.
 
 
+## Upgrading from atticmatic
+
+You can ignore this section if you're not an atticmatic user (the former name
+of borgmatic).
+
+borgmatic only supports Borg now and no longer supports Attic. So if you're
+an Attic user, consider switching to Borg. See the [Borg upgrade
+command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
+for more information. Then, follow the instructions above about setting up
+your borgmatic configuration files.
+
+If you were already using Borg with atticmatic, then you can easily upgrade
+from atticmatic to borgmatic. Simply run the following commands:
+
+    sudo pip uninstall atticmatic
+    sudo pip install borgmatic
+
+That's it! borgmatic will continue using your /etc/borgmatic configuration
+files.
+
 ## Usage
 
 You can run borgmatic and start a backup simply by invoking it without

From 377e3948ff4fba5a7f8dda820020f673447996c1 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 10 Jun 2016 12:15:48 -0700
Subject: [PATCH 105/189] Added tag 1.0.0 for changeset 0e1fbee9358d

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 65c26a7e1..84d6b8ebe 100644
--- a/.hgtags
+++ b/.hgtags
@@ -26,3 +26,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 0000000000000000000000000000000000000000 github/yaml_config_files
 28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master
 dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8
+0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0

From e1e5db22f8f38011c5aed57e09894eec2d1abecf Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 10 Jun 2016 12:34:49 -0700
Subject: [PATCH 106/189] Making a univeral wheel that supports both Python 2
 and 3.

---
 setup.cfg | 3 +++
 setup.py  | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 12871ff0f..4dd514250 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,5 @@
 [metadata]
 description-file=README.md
+
+[bdist_wheel]
+universal=1
diff --git a/setup.py b/setup.py
index f9339766c..19c1c6032 100644
--- a/setup.py
+++ b/setup.py
@@ -27,9 +27,9 @@ setup(
             'borgmatic = borgmatic.command:main',
         ]
     },
-    obsoletes=(
+    obsoletes=[
         'atticmatic',
-    ),
+    ],
     tests_require=(
         'flexmock',
         'pytest',

From 331adca23e49fa40e71efa2602b319334aef9307 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 10 Jun 2016 13:31:37 -0700
Subject: [PATCH 107/189] #19: Support for Borg's --remote-path option to use
 an alternate Borg executable.

---
 NEWS                              |   5 ++
 borgmatic/borg.py                 |  15 ++--
 borgmatic/command.py              |   5 +-
 borgmatic/config.py               |   1 +
 borgmatic/tests/unit/test_borg.py | 144 +++++++++++++++++++++---------
 sample/config                     |  10 ++-
 setup.py                          |   2 +-
 7 files changed, 127 insertions(+), 55 deletions(-)

diff --git a/NEWS b/NEWS
index 7b7dccb6f..af155a906 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+1.0.1
+
+ * #19: Support for Borg's --remote-path option to use an alternate Borg
+   executable. See sample/config.
+
 1.0.0
 
  * Attic is no longer supported, as there hasn't been any recent development on
diff --git a/borgmatic/borg.py b/borgmatic/borg.py
index adde7cfb6..27c570a8c 100644
--- a/borgmatic/borg.py
+++ b/borgmatic/borg.py
@@ -24,7 +24,7 @@ def initialize(storage_config, command=COMMAND):
 
 def create_archive(
     excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND,
-    one_file_system=None
+    one_file_system=None, remote_path=None,
 ):
     '''
     Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
@@ -39,6 +39,7 @@ def create_archive(
     umask = storage_config.get('umask', None)
     umask_flags = ('--umask', str(umask)) if umask else ()
     one_file_system_flags = ('--one-file-system',) if one_file_system else ()
+    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
         VERBOSITY_LOTS: ('--verbose', '--stats'),
@@ -52,7 +53,7 @@ def create_archive(
             timestamp=datetime.now().isoformat(),
         ),
     ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
-        umask_flags + verbosity_flags
+        remote_path_flags + umask_flags + verbosity_flags
 
     subprocess.check_call(full_command)
 
@@ -79,12 +80,13 @@ def _make_prune_flags(retention_config):
     )
 
 
-def prune_archives(verbosity, repository, retention_config, command=COMMAND):
+def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None):
     '''
     Given a verbosity flag, a local or remote repository path, a retention config dict, and a
     command to run, prune attic archives according the the retention policy specified in that
     configuration.
     '''
+    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
         VERBOSITY_LOTS: ('--verbose', '--stats'),
@@ -97,7 +99,7 @@ def prune_archives(verbosity, repository, retention_config, command=COMMAND):
         element
         for pair in _make_prune_flags(retention_config)
         for element in pair
-    ) + verbosity_flags
+    ) + remote_path_flags + verbosity_flags
 
     subprocess.check_call(full_command)
 
@@ -155,7 +157,7 @@ def _make_check_flags(checks, check_last=None):
     ) + last_flag
 
 
-def check_archives(verbosity, repository, consistency_config, command=COMMAND):
+def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None):
     '''
     Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
     command to run, check the contained attic archives for consistency.
@@ -167,6 +169,7 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND):
     if not checks:
         return
 
+    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--verbose',),
         VERBOSITY_LOTS: ('--verbose',),
@@ -175,7 +178,7 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND):
     full_command = (
         command, 'check',
         repository,
-    ) + _make_check_flags(checks, check_last) + verbosity_flags
+    ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
 
     # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
     stdout = None if verbosity_flags else open(os.devnull, 'w')
diff --git a/borgmatic/command.py b/borgmatic/command.py
index 337fc3f5e..bf6713deb 100644
--- a/borgmatic/command.py
+++ b/borgmatic/command.py
@@ -45,13 +45,14 @@ def main():
         args = parse_arguments(*sys.argv[1:])
         config = parse_configuration(args.config_filename, CONFIG_FORMAT)
         repository = config.location['repository']
+        remote_path = config.location['remote_path']
 
         borg.initialize(config.storage)
         borg.create_archive(
             args.excludes_filename, args.verbosity, config.storage, **config.location
         )
-        borg.prune_archives(args.verbosity, repository, config.retention)
-        borg.check_archives(args.verbosity, repository, config.consistency)
+        borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path)
+        borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path)
     except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/borgmatic/config.py b/borgmatic/config.py
index d7a9a0849..36cb4f407 100644
--- a/borgmatic/config.py
+++ b/borgmatic/config.py
@@ -26,6 +26,7 @@ CONFIG_FORMAT = (
         (
             option('source_directories'),
             option('one_file_system', value_type=bool, required=False),
+            option('remote_path', required=False),
             option('repository'),
         ),
     ),
diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py
index 1c032bbb0..051135711 100644
--- a/borgmatic/tests/unit/test_borg.py
+++ b/borgmatic/tests/unit/test_borg.py
@@ -14,8 +14,8 @@ def test_initialize_with_passphrase_should_set_environment():
 
     try:
         os.environ = {}
-        module.initialize({'encryption_passphrase': 'pass'}, command='attic')
-        assert os.environ.get('ATTIC_PASSPHRASE') == 'pass'
+        module.initialize({'encryption_passphrase': 'pass'}, command='borg')
+        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
     finally:
         os.environ = orig_environ
 
@@ -25,8 +25,8 @@ def test_initialize_without_passphrase_should_not_set_environment():
 
     try:
         os.environ = {}
-        module.initialize({}, command='attic')
-        assert os.environ.get('ATTIC_PASSPHRASE') == None
+        module.initialize({}, command='borg')
+        assert os.environ.get('BORG_PASSPHRASE') == None
     finally:
         os.environ = orig_environ
 
@@ -53,11 +53,11 @@ def insert_datetime_mock():
     ).mock
 
 
-CREATE_COMMAND_WITHOUT_EXCLUDES = ('attic', 'create', 'repo::host-now', 'foo', 'bar')
+CREATE_COMMAND_WITHOUT_EXCLUDES = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
 CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes')
 
 
-def test_create_archive_should_call_attic_with_parameters():
+def test_create_archive_should_call_borg_with_parameters():
     insert_subprocess_mock(CREATE_COMMAND)
     insert_platform_mock()
     insert_datetime_mock()
@@ -68,7 +68,7 @@ def test_create_archive_should_call_attic_with_parameters():
         storage_config={},
         source_directories='foo bar',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
@@ -83,11 +83,11 @@ def test_create_archive_with_two_spaces_in_source_directories():
         storage_config={},
         source_directories='foo  bar',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
-def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes():
+def test_create_archive_with_none_excludes_filename_should_call_borg_without_excludes():
     insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
     insert_platform_mock()
     insert_datetime_mock()
@@ -98,11 +98,11 @@ def test_create_archive_with_none_excludes_filename_should_call_attic_without_ex
         storage_config={},
         source_directories='foo bar',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
-def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter():
+def test_create_archive_with_verbosity_some_should_call_borg_with_stats_parameter():
     insert_subprocess_mock(CREATE_COMMAND + ('--stats',))
     insert_platform_mock()
     insert_datetime_mock()
@@ -113,11 +113,11 @@ def test_create_archive_with_verbosity_some_should_call_attic_with_stats_paramet
         storage_config={},
         source_directories='foo bar',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
-def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_parameter():
+def test_create_archive_with_verbosity_lots_should_call_borg_with_verbose_parameter():
     insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -128,11 +128,11 @@ def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_param
         storage_config={},
         source_directories='foo bar',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
-def test_create_archive_with_compression_should_call_attic_with_compression_parameters():
+def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
     insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -143,11 +143,11 @@ def test_create_archive_with_compression_should_call_attic_with_compression_para
         storage_config={'compression': 'rle'},
         source_directories='foo bar',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
-def test_create_archive_with_one_file_system_should_call_attic_with_one_file_system_parameters():
+def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
     insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
     insert_platform_mock()
     insert_datetime_mock()
@@ -158,12 +158,28 @@ def test_create_archive_with_one_file_system_should_call_attic_with_one_file_sys
         storage_config={},
         source_directories='foo bar',
         repository='repo',
-        command='attic',
+        command='borg',
         one_file_system=True,
     )
 
 
-def test_create_archive_with_umask_should_call_attic_with_umask_parameters():
+def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='borg',
+        remote_path='borg1',
+    )
+
+
+def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
     insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -174,12 +190,12 @@ def test_create_archive_with_umask_should_call_attic_with_umask_parameters():
         storage_config={'umask': 740},
         source_directories='foo bar',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
 def test_create_archive_with_source_directories_glob_expands():
-    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
+    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
     flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
@@ -190,12 +206,12 @@ def test_create_archive_with_source_directories_glob_expands():
         storage_config={},
         source_directories='foo*',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo*'))
+    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*'))
     insert_platform_mock()
     insert_datetime_mock()
     flexmock(module).should_receive('glob').with_args('foo*').and_return([])
@@ -206,12 +222,12 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
         storage_config={},
         source_directories='foo*',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
-def test_create_archive_with_glob_should_call_attic_with_expanded_directories():
-    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
+def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
+    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
     flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
@@ -222,7 +238,7 @@ def test_create_archive_with_glob_should_call_attic_with_expanded_directories():
         storage_config={},
         source_directories='foo*',
         repository='repo',
-        command='attic',
+        command='borg',
     )
 
 
@@ -248,11 +264,11 @@ def test_make_prune_flags_should_return_flags_from_config():
 
 
 PRUNE_COMMAND = (
-    'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
+    'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
 )
 
 
-def test_prune_archives_should_call_attic_with_parameters():
+def test_prune_archives_should_call_borg_with_parameters():
     retention_config = flexmock()
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
@@ -263,11 +279,11 @@ def test_prune_archives_should_call_attic_with_parameters():
         verbosity=None,
         repository='repo',
         retention_config=retention_config,
-        command='attic',
+        command='borg',
     )
 
 
-def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_parameter():
+def test_prune_archives_with_verbosity_some_should_call_borg_with_stats_parameter():
     retention_config = flexmock()
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
@@ -278,11 +294,11 @@ def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_paramet
         repository='repo',
         verbosity=VERBOSITY_SOME,
         retention_config=retention_config,
-        command='attic',
+        command='borg',
     )
 
 
-def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
+def test_prune_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter():
     retention_config = flexmock()
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
@@ -293,7 +309,22 @@ def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_param
         repository='repo',
         verbosity=VERBOSITY_LOTS,
         retention_config=retention_config,
-        command='attic',
+        command='borg',
+    )
+
+def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1'))
+
+    module.prune_archives(
+        verbosity=None,
+        repository='repo',
+        retention_config=retention_config,
+        command='borg',
+        remote_path='borg1',
     )
 
 
@@ -345,7 +376,7 @@ def test_make_check_flags_with_last_returns_last_flag():
     assert flags == ('--last', 3)
 
 
-def test_check_archives_should_call_attic_with_parameters():
+def test_check_archives_should_call_borg_with_parameters():
     checks = flexmock()
     check_last = flexmock()
     consistency_config = flexmock().should_receive('get').and_return(check_last).mock
@@ -353,7 +384,7 @@ def test_check_archives_should_call_attic_with_parameters():
     flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
     stdout = flexmock()
     insert_subprocess_mock(
-        ('attic', 'check', 'repo'),
+        ('borg', 'check', 'repo'),
         stdout=stdout, stderr=STDOUT,
     )
     insert_platform_mock()
@@ -365,16 +396,16 @@ def test_check_archives_should_call_attic_with_parameters():
         verbosity=None,
         repository='repo',
         consistency_config=consistency_config,
-        command='attic',
+        command='borg',
     )
 
 
-def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
+def test_check_archives_with_verbosity_some_should_call_borg_with_verbose_parameter():
     consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
-        ('attic', 'check', 'repo', '--verbose'),
+        ('borg', 'check', 'repo', '--verbose'),
         stdout=None, stderr=STDOUT,
     )
     insert_platform_mock()
@@ -384,16 +415,16 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param
         verbosity=VERBOSITY_SOME,
         repository='repo',
         consistency_config=consistency_config,
-        command='attic',
+        command='borg',
     )
 
 
-def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
+def test_check_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter():
     consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
-        ('attic', 'check', 'repo', '--verbose'),
+        ('borg', 'check', 'repo', '--verbose'),
         stdout=None, stderr=STDOUT,
     )
     insert_platform_mock()
@@ -403,7 +434,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param
         verbosity=VERBOSITY_LOTS,
         repository='repo',
         consistency_config=consistency_config,
-        command='attic',
+        command='borg',
     )
 
 
@@ -416,5 +447,30 @@ def test_check_archives_without_any_checks_should_bail():
         verbosity=None,
         repository='repo',
         consistency_config=consistency_config,
-        command='attic',
+        command='borg',
+    )
+
+
+def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
+    checks = flexmock()
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--remote-path', 'borg1'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+    builtins_mock().should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        consistency_config=consistency_config,
+        command='borg',
+        remote_path='borg1',
     )
diff --git a/sample/config b/sample/config
index acea8d7cc..f081e4c4f 100644
--- a/sample/config
+++ b/sample/config
@@ -3,10 +3,12 @@
 # Globs are expanded.
 source_directories: /home /etc /var/log/syslog*
 
-# For Borg only, you can specify to stay in same file system (do not cross
-# mount points).
+# Stay in same file system (do not cross mount points).
 #one_file_system: True
 
+# Alternate Borg remote executable (defaults to "borg"):
+#remote_path: borg1
+
 # Path to local or remote repository.
 repository: user@backupserver:sourcehostname.borg
 
@@ -14,10 +16,12 @@ repository: user@backupserver:sourcehostname.borg
 # Passphrase to unlock the encryption key with. Only use on repositories that
 # were initialized with passphrase/repokey encryption.
 #encryption_passphrase: foo
+
 # Type of compression to use when creating archives. See
 # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
 # for details. Defaults to no compression.
 #compression: lz4
+
 # Umask to be used for borg create.
 #umask: 0740
 
@@ -30,6 +34,7 @@ keep_daily: 7
 keep_weekly: 4
 keep_monthly: 6
 keep_yearly: 1
+
 #prefix: sourcehostname
 
 [consistency]
@@ -38,5 +43,6 @@ keep_yearly: 1
 # checks. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
 # for details.
 checks: repository archives
+
 # Restrict the number of checked archives to the last n.
 #check_last: 3
diff --git a/setup.py b/setup.py
index 19c1c6032..91d6f38f3 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.0.0'
+VERSION = '1.0.1'
 
 
 setup(

From 2e3e68d2cb955532f99251f53b79230350f1d5e9 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 10 Jun 2016 13:34:23 -0700
Subject: [PATCH 108/189] Added tag 1.0.1 for changeset de2d7721cdec

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 84d6b8ebe..58dc3e115 100644
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master
 dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8
 0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0
+de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1

From 600c4389517a533691ed1cf3a11b9af517ad8e4d Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 10 Jun 2016 17:11:28 -0700
Subject: [PATCH 109/189] Reverting to pre-rename issues link, because that
 link isn't yet renamed to borgmatic.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b9e1d9348..c8bd5d924 100644
--- a/README.md
+++ b/README.md
@@ -146,7 +146,7 @@ backups.
 ## Issues and feedback
 
 Got an issue or an idea for a feature enhancement? Check out the [borgmatic
-issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues). In
+issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). In
 order to create a new issue or comment on an issue, you'll need to [login
 first](https://tree.taiga.io/login).
 

From 481dbc14c31ed0f971ffcb53b96b0abfc7582b31 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 12 Jun 2016 22:37:42 -0700
Subject: [PATCH 110/189] Rename issues URL.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c8bd5d924..b9e1d9348 100644
--- a/README.md
+++ b/README.md
@@ -146,7 +146,7 @@ backups.
 ## Issues and feedback
 
 Got an issue or an idea for a feature enhancement? Check out the [borgmatic
-issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). In
+issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues). In
 order to create a new issue or comment on an issue, you'll need to [login
 first](https://tree.taiga.io/login).
 

From 938392b25b31d27230333a180e9e7a8b32e7b65c Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 12 Jun 2016 22:40:04 -0700
Subject: [PATCH 111/189] Restricting issues list to open issues.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b9e1d9348..1bae8c12c 100644
--- a/README.md
+++ b/README.md
@@ -146,7 +146,7 @@ backups.
 ## Issues and feedback
 
 Got an issue or an idea for a feature enhancement? Check out the [borgmatic
-issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues). In
+issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues?page=1&status=399951,399952,399955). In
 order to create a new issue or comment on an issue, you'll need to [login
 first](https://tree.taiga.io/login).
 

From b22b552bf3f11dc2f99c36b4539000e22331c7cf Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 13 Jun 2016 08:53:41 -0700
Subject: [PATCH 112/189] #20: Fix for traceback when remote_path option is
 missing.

---
 NEWS                 | 4 ++++
 borgmatic/command.py | 2 +-
 setup.py             | 2 +-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index af155a906..25d95fd83 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+1.0.2
+
+ * #20: Fix for traceback when remote_path option is missing.
+
 1.0.1
 
  * #19: Support for Borg's --remote-path option to use an alternate Borg
diff --git a/borgmatic/command.py b/borgmatic/command.py
index bf6713deb..784b74a42 100644
--- a/borgmatic/command.py
+++ b/borgmatic/command.py
@@ -45,7 +45,7 @@ def main():
         args = parse_arguments(*sys.argv[1:])
         config = parse_configuration(args.config_filename, CONFIG_FORMAT)
         repository = config.location['repository']
-        remote_path = config.location['remote_path']
+        remote_path = config.location.get('remote_path')
 
         borg.initialize(config.storage)
         borg.create_archive(
diff --git a/setup.py b/setup.py
index 91d6f38f3..55fa95e69 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.0.1'
+VERSION = '1.0.2'
 
 
 setup(

From ead991dcd18eb30934ee09aa42ffcae1057037a8 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 13 Jun 2016 12:02:37 -0700
Subject: [PATCH 113/189] Added tag 1.0.2 for changeset 9603d13910b3

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 58dc3e115..02c0be7da 100644
--- a/.hgtags
+++ b/.hgtags
@@ -28,3 +28,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8
 0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0
 de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
+9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2

From 6bfe524bac6d0a1fd4b3091da964af900eb2d2cb Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Thu, 23 Jun 2016 07:13:25 -0700
Subject: [PATCH 114/189] #21: Fix for verbosity flag not actually causing
 verbose output.

---
 NEWS                              |  4 ++++
 borgmatic/borg.py                 | 12 ++++++------
 borgmatic/tests/unit/test_borg.py | 24 ++++++++++++------------
 setup.py                          |  2 +-
 tox.ini                           |  2 +-
 5 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/NEWS b/NEWS
index 25d95fd83..2cb9b096a 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+1.0.3
+
+ * #21: Fix for verbosity flag not actually causing verbose output.
+
 1.0.2
 
  * #20: Fix for traceback when remote_path option is missing.
diff --git a/borgmatic/borg.py b/borgmatic/borg.py
index 27c570a8c..f4c5ba947 100644
--- a/borgmatic/borg.py
+++ b/borgmatic/borg.py
@@ -41,8 +41,8 @@ def create_archive(
     one_file_system_flags = ('--one-file-system',) if one_file_system else ()
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
-        VERBOSITY_SOME: ('--stats',),
-        VERBOSITY_LOTS: ('--verbose', '--stats'),
+        VERBOSITY_SOME: ('--info', '--stats',),
+        VERBOSITY_LOTS: ('--debug', '--list', '--stats'),
     }.get(verbosity, ())
 
     full_command = (
@@ -88,8 +88,8 @@ def prune_archives(verbosity, repository, retention_config, command=COMMAND, rem
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
-        VERBOSITY_SOME: ('--stats',),
-        VERBOSITY_LOTS: ('--verbose', '--stats'),
+        VERBOSITY_SOME: ('--info', '--stats',),
+        VERBOSITY_LOTS: ('--debug', '--stats'),
     }.get(verbosity, ())
 
     full_command = (
@@ -171,8 +171,8 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND, r
 
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
-        VERBOSITY_SOME: ('--verbose',),
-        VERBOSITY_LOTS: ('--verbose',),
+        VERBOSITY_SOME: ('--info',),
+        VERBOSITY_LOTS: ('--debug',),
     }.get(verbosity, ())
 
     full_command = (
diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py
index 051135711..1d146ba60 100644
--- a/borgmatic/tests/unit/test_borg.py
+++ b/borgmatic/tests/unit/test_borg.py
@@ -102,8 +102,8 @@ def test_create_archive_with_none_excludes_filename_should_call_borg_without_exc
     )
 
 
-def test_create_archive_with_verbosity_some_should_call_borg_with_stats_parameter():
-    insert_subprocess_mock(CREATE_COMMAND + ('--stats',))
+def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
+    insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
     insert_platform_mock()
     insert_datetime_mock()
 
@@ -117,8 +117,8 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_stats_paramete
     )
 
 
-def test_create_archive_with_verbosity_lots_should_call_borg_with_verbose_parameter():
-    insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats'))
+def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
     insert_platform_mock()
     insert_datetime_mock()
 
@@ -283,12 +283,12 @@ def test_prune_archives_should_call_borg_with_parameters():
     )
 
 
-def test_prune_archives_with_verbosity_some_should_call_borg_with_stats_parameter():
+def test_prune_archives_with_verbosity_some_should_call_borg_with_info_parameter():
     retention_config = flexmock()
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--stats',))
+    insert_subprocess_mock(PRUNE_COMMAND + ('--info', '--stats',))
 
     module.prune_archives(
         repository='repo',
@@ -298,12 +298,12 @@ def test_prune_archives_with_verbosity_some_should_call_borg_with_stats_paramete
     )
 
 
-def test_prune_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter():
+def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
     retention_config = flexmock()
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--verbose', '--stats',))
+    insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats',))
 
     module.prune_archives(
         repository='repo',
@@ -400,12 +400,12 @@ def test_check_archives_should_call_borg_with_parameters():
     )
 
 
-def test_check_archives_with_verbosity_some_should_call_borg_with_verbose_parameter():
+def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
     consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--verbose'),
+        ('borg', 'check', 'repo', '--info'),
         stdout=None, stderr=STDOUT,
     )
     insert_platform_mock()
@@ -419,12 +419,12 @@ def test_check_archives_with_verbosity_some_should_call_borg_with_verbose_parame
     )
 
 
-def test_check_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter():
+def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
     consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--verbose'),
+        ('borg', 'check', 'repo', '--debug'),
         stdout=None, stderr=STDOUT,
     )
     insert_platform_mock()
diff --git a/setup.py b/setup.py
index 55fa95e69..d4dbbbe93 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.0.2'
+VERSION = '1.0.3'
 
 
 setup(
diff --git a/tox.ini b/tox.ini
index 64d1f6c1f..90c5ce21c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,4 +5,4 @@ skipsdist=True
 [testenv]
 usedevelop=True
 deps=-rtest_requirements.txt
-commands = py.test []
+commands = py.test borgmatic []

From f6d2e983d989a89308654547b9ffde7eb004d0d2 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Thu, 23 Jun 2016 07:13:29 -0700
Subject: [PATCH 115/189] Added tag 1.0.3 for changeset 32c6341dda9f

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 02c0be7da..06c448a4b 100644
--- a/.hgtags
+++ b/.hgtags
@@ -29,3 +29,4 @@ dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8
 0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0
 de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
 9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2
+32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3

From 5bd1cc55804ae333dc264942b11fa58b506a9ef6 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 3 Jul 2016 22:07:53 -0700
Subject: [PATCH 116/189] #18: Fix for README mention of sample files not
 included in package. Also, added logo.

---
 NEWS                 |  5 +++++
 README.md            | 24 ++++++++++++++++++++----
 setup.py             |  2 +-
 static/borgmatic.svg |  1 +
 4 files changed, 27 insertions(+), 5 deletions(-)
 create mode 100644 static/borgmatic.svg

diff --git a/NEWS b/NEWS
index 2cb9b096a..724a7c727 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+1.0.3-dev
+
+ * #18: Fix for README mention of sample files not included in package.
+ * Added logo.
+
 1.0.3
 
  * #21: Fix for verbosity flag not actually causing verbose output.
diff --git a/README.md b/README.md
index 1bae8c12c..00c2b97a4 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,6 @@
-title: Borgmatic
+title: borgmatic
+
+<img src="static/borgmatic.svg" alt="borgmatic logo" style="width: 8em; float: right; padding-left: 1em;" />
 
 ## Overview
 
@@ -57,11 +59,14 @@ To install borgmatic, run the following command to download and install it:
 
     sudo pip install --upgrade borgmatic
 
-Then, copy the following configuration files:
+Then, download a [sample config
+file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config) and a
+[sample excludes
+file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/excludes). From the
+directory where you downloaded them:
 
-    sudo cp sample/borgmatic.cron /etc/cron.d/borgmatic
     sudo mkdir /etc/borgmatic/
-    sudo cp sample/config sample/excludes /etc/borgmatic/
+    sudo mv config excludes /etc/borgmatic/
 
 Lastly, modify the /etc files with your desired configuration.
 
@@ -111,6 +116,17 @@ If you'd like to see the available command-line arguments, view the help:
     borgmatic --help
 
 
+## Autopilot
+
+If you want to run borgmatic automatically, say once a day, the you can
+configure a job runner like cron to invoke it periodically. Download the
+[sample cron
+file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/borgmatic.cron).
+Then, from the directory where you downloaded it:
+
+    sudo cp borgmatic.cron /etc/cron.d/borgmatic
+
+
 ## Running tests
 
 First install tox, which is used for setting up testing environments:
diff --git a/setup.py b/setup.py
index d4dbbbe93..63b00818f 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.0.3'
+VERSION = '1.0.3-dev'
 
 
 setup(
diff --git a/static/borgmatic.svg b/static/borgmatic.svg
new file mode 100644
index 000000000..1c308168f
--- /dev/null
+++ b/static/borgmatic.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" x="0px" y="0px" viewBox="0 0 100 100" enable-background="new 0 0 100 100" xml:space="preserve"><g><circle cx="63.461" cy="59.615" r="5.626"></circle><path d="M81.432,48.015c0.779-10.849-1.758-19.894-7.407-26.293C68.447,15.403,59.915,11.923,50,11.923   c-6.729,0-12.834,1.645-17.81,4.724c-0.405-0.06-0.823-0.073-1.248,0.005C8.128,20.817,0,34.623,0,46.154   c0,6.155,1.856,11.005,5.519,14.417c2.082,1.941,4.461,3.223,6.659,3.921C13.041,74.256,18.345,81,23.077,81h1.438   C30.168,94,44.847,99.614,50,99.614c6.708,0,24.493-7.875,27.38-25.392c4.758-1.888,10.697-7.373,10.697-13.711   C88.077,54.639,85.395,49.855,81.432,48.015z M7.691,46.154c0-3.397,1.238-13.399,14.747-19.083   c-2.043,3.821-3.31,8.234-3.771,13.136c-3.06,2.521-5.577,7.792-6.366,15.873c-0.534-0.316-1.056-0.686-1.538-1.135   C8.725,53.045,7.691,50.088,7.691,46.154z M74.23,67.947c-1.806,0-3.309,1.389-3.451,3.188C69.608,85.928,53.914,92.575,50,92.575   c-3.275,0-14.607-4.474-19.03-14.325c1.001-0.709,2.168-1.25,3.646-1.25c3.809,0,3.847,0,3.847,0c2.126,0,7.692-4.023,7.692-19.494   C46.154,46.241,40.588,42,38.462,42c0,0-0.195,0-3.847,0c-3.65,0-5.566-4-7.692-4h-1.046c0.813-4,2.646-8.756,5.473-11.94   c4.321-4.87,10.771-7.329,18.65-7.329c8.016,0,14.528,2.636,18.835,7.515c4.821,5.462,6.697,13.762,5.424,23.95   c-0.123,0.984,0.183,1.99,0.84,2.734s1.602,1.179,2.595,1.179c1.671,0,3.46,2.562,3.46,6.371   C81.153,64.016,75.754,67.812,74.23,67.947z M30.432,59.615c0-3.377,2.734-6.107,6.106-6.107c3.373,0,6.107,2.73,6.107,6.107   s-2.734,6.107-6.107,6.107C33.166,65.723,30.432,62.992,30.432,59.615z"></path></g></svg>
\ No newline at end of file

From abb6bed4598206f4faa3abc5f7d1605b2ba4c4b4 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 4 Jul 2016 09:19:34 -0700
Subject: [PATCH 117/189] Sample files for triggering borgmatic from a systemd
 timer.

---
 NEWS                                      |  1 +
 README.md                                 | 29 +++++++++++++++++++----
 sample/{borgmatic.cron => cron/borgmatic} |  0
 sample/systemd/borgmatic.service          |  6 +++++
 sample/systemd/borgmatic.timer            |  8 +++++++
 5 files changed, 40 insertions(+), 4 deletions(-)
 rename sample/{borgmatic.cron => cron/borgmatic} (100%)
 create mode 100644 sample/systemd/borgmatic.service
 create mode 100644 sample/systemd/borgmatic.timer

diff --git a/NEWS b/NEWS
index 724a7c727..3dc98c5b0 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,7 @@
 1.0.3-dev
 
  * #18: Fix for README mention of sample files not included in package.
+ * #22: Sample files for triggering borgmatic from a systemd timer.
  * Added logo.
 
 1.0.3
diff --git a/README.md b/README.md
index 00c2b97a4..4d7f18ee1 100644
--- a/README.md
+++ b/README.md
@@ -119,12 +119,33 @@ If you'd like to see the available command-line arguments, view the help:
 ## Autopilot
 
 If you want to run borgmatic automatically, say once a day, the you can
-configure a job runner like cron to invoke it periodically. Download the
-[sample cron
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/borgmatic.cron).
+configure a job runner to invoke it periodically.
+
+### cron
+
+If you're using cron, download the [sample cron
+file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/cron/borgmatic).
 Then, from the directory where you downloaded it:
 
-    sudo cp borgmatic.cron /etc/cron.d/borgmatic
+    sudo mv borgmatic /etc/cron.d/borgmatic
+
+You can modify the cron file if you'd like to run borgmatic more or less frequently.
+
+### systemd
+
+If you're using systemd instead of cron to run jobs, download the [sample
+systemd service
+file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.service)
+and the [sample systemd timer
+file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.timer).
+Then, from the directory where you downloaded them:
+
+    sudo mv {borgmatic.service,borgmatic.timer} /etc/systemd/system/
+    sudo systemctl enable borgmatic.timer
+    sudo systemctl start borgmatic.timer
+
+Feel free to modify the timer file based on how frequently you'd like
+borgmatic to run.
 
 
 ## Running tests
diff --git a/sample/borgmatic.cron b/sample/cron/borgmatic
similarity index 100%
rename from sample/borgmatic.cron
rename to sample/cron/borgmatic
diff --git a/sample/systemd/borgmatic.service b/sample/systemd/borgmatic.service
new file mode 100644
index 000000000..443b1f0d8
--- /dev/null
+++ b/sample/systemd/borgmatic.service
@@ -0,0 +1,6 @@
+[Unit]
+Description=borgmatic backup
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/borgmatic
diff --git a/sample/systemd/borgmatic.timer b/sample/systemd/borgmatic.timer
new file mode 100644
index 000000000..4b49d1f77
--- /dev/null
+++ b/sample/systemd/borgmatic.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Run borgmatic backup
+
+[Timer]
+OnCalendar=daily
+
+[Install]
+WantedBy=timers.target

From 87c65fb7239706920523162391c01aa3c9d43e6b Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 4 Jul 2016 09:35:51 -0700
Subject: [PATCH 118/189] Removing unnecessary curlies from bash command.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4d7f18ee1..a8a0a0328 100644
--- a/README.md
+++ b/README.md
@@ -140,7 +140,7 @@ and the [sample systemd timer
 file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.timer).
 Then, from the directory where you downloaded them:
 
-    sudo mv {borgmatic.service,borgmatic.timer} /etc/systemd/system/
+    sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
     sudo systemctl enable borgmatic.timer
     sudo systemctl start borgmatic.timer
 

From 1aaf27dfb218c155b9304247ee94d1435dd99f11 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 25 Jun 2017 10:36:36 -0700
Subject: [PATCH 119/189] Changed example umask config to be more realistic.

---
 sample/config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sample/config b/sample/config
index f081e4c4f..e778aaacd 100644
--- a/sample/config
+++ b/sample/config
@@ -23,7 +23,7 @@ repository: user@backupserver:sourcehostname.borg
 #compression: lz4
 
 # Umask to be used for borg create.
-#umask: 0740
+#umask: 0077
 
 [retention]
 # Retention policy for how many backups to keep in each category. See

From e00f74ddf704201e8c110b6f9681194f442561c5 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 2 Jul 2017 17:18:33 -0700
Subject: [PATCH 120/189] Dropped Python 2 support. Now Python 3 only.

---
 NEWS                                       | 1 +
 README.md                                  | 3 +++
 borgmatic/config.py                        | 8 +-------
 borgmatic/tests/builtins.py                | 7 +------
 borgmatic/tests/integration/test_config.py | 7 +------
 tox.ini                                    | 2 +-
 6 files changed, 8 insertions(+), 20 deletions(-)

diff --git a/NEWS b/NEWS
index 3dc98c5b0..912436059 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,7 @@
 
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.
+ * Dropped Python 2 support. Now Python 3 only.
  * Added logo.
 
 1.0.3
diff --git a/README.md b/README.md
index a8a0a0328..7eb0eb1b5 100644
--- a/README.md
+++ b/README.md
@@ -59,6 +59,9 @@ To install borgmatic, run the following command to download and install it:
 
     sudo pip install --upgrade borgmatic
 
+Make sure you're using Python 3, as borgmatic does not support Python 2. (You
+may have to use "pip3" instead of "pip".)
+
 Then, download a [sample config
 file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config) and a
 [sample excludes
diff --git a/borgmatic/config.py b/borgmatic/config.py
index 36cb4f407..9c455b3e9 100644
--- a/borgmatic/config.py
+++ b/borgmatic/config.py
@@ -1,11 +1,5 @@
 from collections import OrderedDict, namedtuple
-
-try:
-    # Python 2
-    from ConfigParser import RawConfigParser
-except ImportError:
-    # Python 3
-    from configparser import RawConfigParser
+from configparser import RawConfigParser
 
 
 Section_format = namedtuple('Section_format', ('name', 'options'))
diff --git a/borgmatic/tests/builtins.py b/borgmatic/tests/builtins.py
index ff48f939b..53970cdee 100644
--- a/borgmatic/tests/builtins.py
+++ b/borgmatic/tests/builtins.py
@@ -3,9 +3,4 @@ import sys
 
 
 def builtins_mock():
-    try:
-        # Python 2
-        return flexmock(sys.modules['__builtin__'])
-    except KeyError:
-        # Python 3
-        return flexmock(sys.modules['builtins'])
+    return flexmock(sys.modules['builtins'])
diff --git a/borgmatic/tests/integration/test_config.py b/borgmatic/tests/integration/test_config.py
index e849b4f76..56cad6c34 100644
--- a/borgmatic/tests/integration/test_config.py
+++ b/borgmatic/tests/integration/test_config.py
@@ -1,9 +1,4 @@
-try:
-    # Python 2
-    from cStringIO import StringIO
-except ImportError:
-    # Python 3
-    from io import StringIO
+from io import StringIO
 
 from collections import OrderedDict
 import string
diff --git a/tox.ini b/tox.ini
index 90c5ce21c..579a91956 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist=py27,py34
+envlist=py34
 skipsdist=True
 
 [testenv]

From 6e85940d6308ea7fe03457a8a00b7298ad57e0fb Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 4 Jul 2017 16:52:24 -0700
Subject: [PATCH 121/189] Basic YAML configuration file parsing.

---
 .hgignore                                     |  1 +
 MANIFEST.in                                   |  1 +
 NEWS                                          |  2 +-
 borgmatic/command.py                          |  2 +-
 borgmatic/config/__init__.py                  |  0
 borgmatic/{config.py => config/legacy.py}     |  0
 borgmatic/config/schema.yaml                  | 48 +++++++++++
 borgmatic/config/yaml.py                      | 84 +++++++++++++++++++
 borgmatic/tests/builtins.py                   |  6 --
 .../tests/integration/config/__init__.py      |  0
 .../{test_config.py => config/test_legacy.py} |  2 +-
 borgmatic/tests/unit/config/__init__.py       |  0
 .../{test_config.py => config/test_legacy.py} |  2 +-
 borgmatic/tests/unit/test_borg.py             |  6 +-
 sample/config.yaml                            | 54 ++++++++++++
 setup.py                                      | 10 ++-
 16 files changed, 203 insertions(+), 15 deletions(-)
 create mode 100644 MANIFEST.in
 create mode 100644 borgmatic/config/__init__.py
 rename borgmatic/{config.py => config/legacy.py} (100%)
 create mode 100644 borgmatic/config/schema.yaml
 create mode 100644 borgmatic/config/yaml.py
 delete mode 100644 borgmatic/tests/builtins.py
 create mode 100644 borgmatic/tests/integration/config/__init__.py
 rename borgmatic/tests/integration/{test_config.py => config/test_legacy.py} (92%)
 create mode 100644 borgmatic/tests/unit/config/__init__.py
 rename borgmatic/tests/unit/{test_config.py => config/test_legacy.py} (99%)
 create mode 100644 sample/config.yaml

diff --git a/.hgignore b/.hgignore
index 41280764d..f41ba9356 100644
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,7 @@ syntax: glob
 *.egg-info
 *.pyc
 *.swp
+.cache
 .tox
 build
 dist
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 000000000..20ff23126
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1 @@
+include borgmatic/config/schema.yaml
diff --git a/NEWS b/NEWS
index 912436059..b5d421636 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
-1.0.3-dev
+1.1.0
 
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.
diff --git a/borgmatic/command.py b/borgmatic/command.py
index 784b74a42..ef7bbee6f 100644
--- a/borgmatic/command.py
+++ b/borgmatic/command.py
@@ -5,7 +5,7 @@ from subprocess import CalledProcessError
 import sys
 
 from borgmatic import borg
-from borgmatic.config import parse_configuration, CONFIG_FORMAT
+from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT
 
 
 DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'
diff --git a/borgmatic/config/__init__.py b/borgmatic/config/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/borgmatic/config.py b/borgmatic/config/legacy.py
similarity index 100%
rename from borgmatic/config.py
rename to borgmatic/config/legacy.py
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
new file mode 100644
index 000000000..83c50bb19
--- /dev/null
+++ b/borgmatic/config/schema.yaml
@@ -0,0 +1,48 @@
+map:
+    location:
+        required: True
+        map:
+            source_directories:
+                required: True
+                seq:
+                    - type: scalar
+            one_file_system:
+                type: bool
+            remote_path:
+                type: scalar
+            repository:
+                required: True
+                type: scalar
+    storage:
+        map:
+            encryption_passphrase:
+                type: scalar
+            compression:
+                type: scalar
+            umask:
+                type: scalar
+    retention:
+        map:
+            keep_within:
+                type: scalar
+            keep_hourly:
+                type: int
+            keep_daily:
+                type: int
+            keep_weekly:
+                type: int
+            keep_monthly:
+                type: int
+            keep_yearly:
+                type: int
+            prefix:
+                type: scalar
+    consistency:
+        map:
+            checks:
+                seq:
+                    - type: str
+                      enum: ['repository', 'archives', 'disabled']
+                      unique: True
+            check_last:
+                type: int
diff --git a/borgmatic/config/yaml.py b/borgmatic/config/yaml.py
new file mode 100644
index 000000000..4ff01b98d
--- /dev/null
+++ b/borgmatic/config/yaml.py
@@ -0,0 +1,84 @@
+import logging
+import sys
+import warnings
+
+import pkg_resources
+import pykwalify.core
+import pykwalify.errors
+import ruamel.yaml.error
+
+
+def schema_filename():
+    '''
+    Path to the installed YAML configuration schema file, used to validate and parse the
+    configuration.
+    '''
+    return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
+
+
+class Validation_error(ValueError):
+    '''
+    A collection of error message strings generated when attempting to validate a particular
+    configurartion file.
+    '''
+    def __init__(self, config_filename, error_messages):
+        self.config_filename = config_filename
+        self.error_messages = error_messages
+
+
+def parse_configuration(config_filename, schema_filename):
+    '''
+    Given the path to a config filename in YAML format and the path to a schema filename in
+    pykwalify YAML schema format, return the parsed configuration as a data structure of nested
+    dicts and lists corresponding to the schema. Example return value:
+
+       {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
+       'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
+
+    Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
+    have permissions to read the file, or Validation_error if the config does not match the schema.
+    '''
+    warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
+    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
+
+    try:
+        validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
+    except pykwalify.errors.CoreError as error:
+        if 'do not exists on disk' in str(error):
+            raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
+        if 'Unable to load any data' in str(error):
+            # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
+            # So reach back to the originating exception from ruamel.yaml for something more useful.
+            raise Validation_error(config_filename, (error.__context__,))
+        raise
+
+    parsed_result = validator.validate(raise_exception=False)
+
+    if validator.validation_errors:
+        raise Validation_error(config_filename, validator.validation_errors)
+
+    return parsed_result
+
+
+def display_validation_error(validation_error):
+    '''
+    Given a Validation_error, display its error messages to stderr.
+    '''
+    print(
+        'An error occurred while parsing a configuration file at {}:'.format(
+            validation_error.config_filename
+        ),
+        file=sys.stderr,
+    )
+
+    for error in validation_error.error_messages:
+        print(error, file=sys.stderr)
+
+
+# FOR TESTING
+if __name__ == '__main__':
+    try:
+        configuration = parse_configuration('sample/config.yaml', schema_filename())
+        print(configuration)
+    except Validation_error as error:
+        display_validation_error(error)
diff --git a/borgmatic/tests/builtins.py b/borgmatic/tests/builtins.py
deleted file mode 100644
index 53970cdee..000000000
--- a/borgmatic/tests/builtins.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from flexmock import flexmock
-import sys
-
-
-def builtins_mock():
-    return flexmock(sys.modules['builtins'])
diff --git a/borgmatic/tests/integration/config/__init__.py b/borgmatic/tests/integration/config/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/borgmatic/tests/integration/test_config.py b/borgmatic/tests/integration/config/test_legacy.py
similarity index 92%
rename from borgmatic/tests/integration/test_config.py
rename to borgmatic/tests/integration/config/test_legacy.py
index 56cad6c34..13ee65465 100644
--- a/borgmatic/tests/integration/test_config.py
+++ b/borgmatic/tests/integration/config/test_legacy.py
@@ -3,7 +3,7 @@ from io import StringIO
 from collections import OrderedDict
 import string
 
-from borgmatic import config as module
+from borgmatic.config import legacy as module
 
 
 def test_parse_section_options_with_punctuation_should_return_section_options():
diff --git a/borgmatic/tests/unit/config/__init__.py b/borgmatic/tests/unit/config/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/borgmatic/tests/unit/test_config.py b/borgmatic/tests/unit/config/test_legacy.py
similarity index 99%
rename from borgmatic/tests/unit/test_config.py
rename to borgmatic/tests/unit/config/test_legacy.py
index 01e21bb0f..9e95cac6c 100644
--- a/borgmatic/tests/unit/test_config.py
+++ b/borgmatic/tests/unit/config/test_legacy.py
@@ -3,7 +3,7 @@ from collections import OrderedDict
 from flexmock import flexmock
 import pytest
 
-from borgmatic import config as module
+from borgmatic.config import legacy as module
 
 
 def test_option_should_create_config_option():
diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py
index 1d146ba60..3c2aa1c71 100644
--- a/borgmatic/tests/unit/test_borg.py
+++ b/borgmatic/tests/unit/test_borg.py
@@ -1,11 +1,11 @@
 from collections import OrderedDict
 from subprocess import STDOUT
+import sys
 import os
 
 from flexmock import flexmock
 
 from borgmatic import borg as module
-from borgmatic.tests.builtins import builtins_mock
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
@@ -389,7 +389,7 @@ def test_check_archives_should_call_borg_with_parameters():
     )
     insert_platform_mock()
     insert_datetime_mock()
-    builtins_mock().should_receive('open').and_return(stdout)
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
     flexmock(module.os).should_receive('devnull')
 
     module.check_archives(
@@ -464,7 +464,7 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
     )
     insert_platform_mock()
     insert_datetime_mock()
-    builtins_mock().should_receive('open').and_return(stdout)
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
     flexmock(module.os).should_receive('devnull')
 
     module.check_archives(
diff --git a/sample/config.yaml b/sample/config.yaml
new file mode 100644
index 000000000..6ca27c723
--- /dev/null
+++ b/sample/config.yaml
@@ -0,0 +1,54 @@
+location:
+    # List of source directories to backup. Globs are expanded.
+    source_directories:
+        - /home
+        - /etc
+        - /var/log/syslog*
+
+    # Stay in same file system (do not cross mount points).
+    #one_file_system: yes
+
+    # Alternate Borg remote executable (defaults to "borg"):
+    #remote_path: borg1
+
+    # Path to local or remote repository.
+    repository: user@backupserver:sourcehostname.borg
+
+#storage:
+    # Passphrase to unlock the encryption key with. Only use on repositories
+    # that were initialized with passphrase/repokey encryption.
+    #encryption_passphrase: foo
+
+    # Type of compression to use when creating archives. See
+    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
+    # for details. Defaults to no compression.
+    #compression: lz4
+
+    # Umask to be used for borg create.
+    #umask: 0077
+
+retention:
+    # Retention policy for how many backups to keep in each category. See
+    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for
+    # details.
+    #keep_within: 3H
+    #keep_hourly: 24
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6
+    keep_yearly: 1
+
+    # When pruning, only consider archive names starting with this prefix.
+    #prefix: sourcehostname
+
+consistency:
+    # List of consistency checks to run: "repository", "archives", or both.
+    # Defaults to both. Set to "disabled" to disable all consistency checks.
+    # See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
+    # for details.
+    checks:
+        - repository
+        - archives
+
+    # Restrict the number of checked archives to the last n.
+    #check_last: 3
diff --git a/setup.py b/setup.py
index 63b00818f..198a2d55f 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.0.3-dev'
+VERSION = '1.1.0'
 
 
 setup(
@@ -30,8 +30,14 @@ setup(
     obsoletes=[
         'atticmatic',
     ],
+    install_requires=(
+        'pykwalify',
+        'ruamel.yaml<=0.15',
+        'setuptools',
+    ),
     tests_require=(
         'flexmock',
         'pytest',
-    )
+    ),
+    include_package_data=True,
 )

From 1dc60d2856445564b2508db24e2e2399e7861203 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 4 Jul 2017 18:23:59 -0700
Subject: [PATCH 122/189] Integrating YAML config into borgmatic and updating
 README.

---
 README.md            | 38 +++++++++++++++++++++-----------------
 borgmatic/command.py |  8 ++++----
 2 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index 7eb0eb1b5..5e69283ae 100644
--- a/README.md
+++ b/README.md
@@ -13,24 +13,28 @@ all on the command-line, and handles common errors.
 
 Here's an example config file:
 
-```INI
-[location]
-# Space-separated list of source directories to backup.
-# Globs are expanded.
-source_directories: /home /etc /var/log/syslog*
+```yaml
+location:
+    # List of source directories to backup. Globs are expanded.
+    source_directories:
+        - /home
+        - /etc
+        - /var/log/syslog*
 
-# Path to local or remote backup repository.
-repository: user@backupserver:sourcehostname.borg
+    # Path to local or remote repository.
+    repository: user@backupserver:sourcehostname.borg
 
-[retention]
-# Retention policy for how many backups to keep in each category.
-keep_daily: 7
-keep_weekly: 4
-keep_monthly: 6
+retention:
+    # Retention policy for how many backups to keep in each category.
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6
 
-[consistency]
-# Consistency checks to run, or "disabled" to prevent checks.
-checks: repository archives
+consistency:
+    # List of consistency checks to run: "repository", "archives", or both.
+    checks:
+        - repository
+        - archives
 ```
 
 Additionally, exclude patterns can be specified in a separate excludes config
@@ -63,13 +67,13 @@ Make sure you're using Python 3, as borgmatic does not support Python 2. (You
 may have to use "pip3" instead of "pip".)
 
 Then, download a [sample config
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config) and a
+file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config.yaml) and a
 [sample excludes
 file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/excludes). From the
 directory where you downloaded them:
 
     sudo mkdir /etc/borgmatic/
-    sudo mv config excludes /etc/borgmatic/
+    sudo mv config.yaml excludes /etc/borgmatic/
 
 Lastly, modify the /etc files with your desired configuration.
 
diff --git a/borgmatic/command.py b/borgmatic/command.py
index ef7bbee6f..cba6ef3af 100644
--- a/borgmatic/command.py
+++ b/borgmatic/command.py
@@ -5,10 +5,10 @@ from subprocess import CalledProcessError
 import sys
 
 from borgmatic import borg
-from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT
+from borgmatic.config.yaml import parse_configuration, schema_filename
 
 
-DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'
+DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 
 
@@ -43,7 +43,7 @@ def parse_arguments(*arguments):
 def main():
     try:
         args = parse_arguments(*sys.argv[1:])
-        config = parse_configuration(args.config_filename, CONFIG_FORMAT)
+        config = parse_configuration(args.config_filename, schema_filename())
         repository = config.location['repository']
         remote_path = config.location.get('remote_path')
 
@@ -53,6 +53,6 @@ def main():
         )
         borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path)
         borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path)
-    except (ValueError, IOError, CalledProcessError) as error:
+    except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

From bff6980eeedb2548198202f39f4799f00c6f2045 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 4 Jul 2017 18:32:37 -0700
Subject: [PATCH 123/189] Tests for YAML config code.

---
 .../tests/integration/config/test_yaml.py     | 113 ++++++++++++++++++
 1 file changed, 113 insertions(+)
 create mode 100644 borgmatic/tests/integration/config/test_yaml.py

diff --git a/borgmatic/tests/integration/config/test_yaml.py b/borgmatic/tests/integration/config/test_yaml.py
new file mode 100644
index 000000000..ebd33a369
--- /dev/null
+++ b/borgmatic/tests/integration/config/test_yaml.py
@@ -0,0 +1,113 @@
+import io
+import string
+import sys
+import os
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic.config import yaml as module
+
+
+def test_schema_filename_returns_plausable_path():
+    schema_path = module.schema_filename()    
+
+    assert schema_path.endswith('/schema.yaml')
+
+
+def mock_config_and_schema(config_yaml):
+    '''
+    Set up mocks for the config config YAML string and the default schema so that pykwalify consumes
+    them when parsing the configuration. This is a little brittle in that it's relying on pykwalify
+    to open() the respective files in a particular order.
+    '''
+    config_stream = io.StringIO(config_yaml)
+    schema_stream = open(module.schema_filename())
+    builtins = flexmock(sys.modules['builtins']).should_call('open').mock
+    builtins.should_receive('open').and_return(config_stream).and_return(schema_stream)
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+
+def test_parse_configuration_transforms_file_into_mapping():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+                - /etc
+
+            repository: hostname.borg
+
+        retention:
+            keep_daily: 7
+
+        consistency:
+            checks:
+                - repository
+                - archives
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
+        'retention': {'keep_daily': 7},
+        'consistency': {'checks': ['repository', 'archives']},
+    }
+
+
+def test_parse_configuration_passes_through_quoted_punctuation():
+    escaped_punctuation = string.punctuation.replace('\\', r'\\').replace('"', r'\"')
+
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repository: "{}.borg"
+        '''.format(escaped_punctuation)
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {
+            'source_directories': ['/home'],
+            'repository': '{}.borg'.format(string.punctuation),
+        },
+    }
+
+
+def test_parse_configuration_raises_for_missing_config_file():
+    with pytest.raises(FileNotFoundError):
+        module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_missing_schema_file():
+    mock_config_and_schema('')
+    flexmock(os.path).should_receive('exists').with_args('schema.yaml').and_return(False)
+
+    with pytest.raises(FileNotFoundError):
+        module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_syntax_error():
+    mock_config_and_schema('invalid = yaml')
+
+    with pytest.raises(module.Validation_error):
+        module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_validation_error():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories: yes
+            repository: hostname.borg
+        '''
+    )
+
+    with pytest.raises(module.Validation_error):
+        module.parse_configuration('config.yaml', 'schema.yaml')

From 745de200dfcc43555e3f38b4bd6c1d5671dfa9e5 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 8 Jul 2017 22:33:51 -0700
Subject: [PATCH 124/189] Basic YAML generating / validating / converting to.

---
 NEWS                                          |  5 +-
 borgmatic/commands/__init__.py                |  0
 .../{command.py => commands/borgmatic.py}     |  7 +-
 borgmatic/commands/convert_config.py          | 54 +++++++++++
 borgmatic/config/convert.py                   | 41 +++++++++
 borgmatic/config/generate.py                  | 90 +++++++++++++++++++
 borgmatic/config/schema.yaml                  | 62 +++++++++++++
 borgmatic/config/{yaml.py => validate.py}     | 33 +++----
 .../tests/integration/commands/__init__.py    |  0
 .../test_borgmatic.py}                        |  2 +-
 .../config/{test_yaml.py => test_validate.py} | 13 +--
 borgmatic/tests/unit/config/test_convert.py   | 44 +++++++++
 sample/config.yaml                            |  6 +-
 setup.py                                      |  5 +-
 test_requirements.txt                         |  3 +
 tox.ini                                       |  2 +-
 16 files changed, 327 insertions(+), 40 deletions(-)
 create mode 100644 borgmatic/commands/__init__.py
 rename borgmatic/{command.py => commands/borgmatic.py} (83%)
 create mode 100644 borgmatic/commands/convert_config.py
 create mode 100644 borgmatic/config/convert.py
 create mode 100644 borgmatic/config/generate.py
 rename borgmatic/config/{yaml.py => validate.py} (65%)
 create mode 100644 borgmatic/tests/integration/commands/__init__.py
 rename borgmatic/tests/integration/{test_command.py => commands/test_borgmatic.py} (97%)
 rename borgmatic/tests/integration/config/{test_yaml.py => test_validate.py} (88%)
 create mode 100644 borgmatic/tests/unit/config/test_convert.py

diff --git a/NEWS b/NEWS
index b5d421636..3d16ba536 100644
--- a/NEWS
+++ b/NEWS
@@ -1,8 +1,9 @@
-1.1.0
+1.1.0.dev0
 
+ * Switched config file format to YAML. Run convert-borgmatic-config to upgrade.
+ * Dropped Python 2 support. Now Python 3 only.
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.
- * Dropped Python 2 support. Now Python 3 only.
  * Added logo.
 
 1.0.3
diff --git a/borgmatic/commands/__init__.py b/borgmatic/commands/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/borgmatic/command.py b/borgmatic/commands/borgmatic.py
similarity index 83%
rename from borgmatic/command.py
rename to borgmatic/commands/borgmatic.py
index cba6ef3af..c817976a8 100644
--- a/borgmatic/command.py
+++ b/borgmatic/commands/borgmatic.py
@@ -5,7 +5,7 @@ from subprocess import CalledProcessError
 import sys
 
 from borgmatic import borg
-from borgmatic.config.yaml import parse_configuration, schema_filename
+from borgmatic.config.validate import parse_configuration, schema_filename
 
 
 DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
@@ -14,9 +14,8 @@ DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 
 def parse_arguments(*arguments):
     '''
-    Given the name of the command with which this script was invoked and command-line arguments,
-    parse the arguments and return them as an ArgumentParser instance. Use the command name to
-    determine the default configuration and excludes paths.
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
     '''
     parser = ArgumentParser()
     parser.add_argument(
diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py
new file mode 100644
index 000000000..8f645c0aa
--- /dev/null
+++ b/borgmatic/commands/convert_config.py
@@ -0,0 +1,54 @@
+from __future__ import print_function
+from argparse import ArgumentParser
+import os
+from subprocess import CalledProcessError
+import sys
+
+from ruamel import yaml
+
+from borgmatic import borg
+from borgmatic.config import convert, generate, legacy, validate
+
+
+DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
+DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
+DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.')
+    parser.add_argument(
+        '-s', '--source',
+        dest='source_filename',
+        default=DEFAULT_SOURCE_CONFIG_FILENAME,
+        help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
+    )
+    parser.add_argument(
+        '-d', '--destination',
+        dest='destination_filename',
+        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
+        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():
+    try:
+        args = parse_arguments(*sys.argv[1:])
+        source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
+        schema = yaml.round_trip_load(open(validate.schema_filename()).read())
+
+        destination_config = convert.convert_legacy_parsed_config(source_config, schema)
+
+        generate.write_configuration(args.destination_filename, destination_config)
+
+        # TODO: As a backstop, check that the written config can actually be read and parsed, and
+        # that it matches the destination config data structure that was written.
+    except (ValueError, OSError) as error:
+        print(error, file=sys.stderr)
+        sys.exit(1)
diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py
new file mode 100644
index 000000000..72b9ba314
--- /dev/null
+++ b/borgmatic/config/convert.py
@@ -0,0 +1,41 @@
+from ruamel import yaml
+
+from borgmatic.config import generate
+
+
+def _convert_section(source_section_config, section_schema):
+    '''
+    Given a legacy Parsed_config instance for a single section, convert it to its corresponding
+    yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.
+
+    Additionally, use the section schema as a source of helpful comments to include within the
+    returned CommentedMap.
+    '''
+    destination_section_config = yaml.comments.CommentedMap(source_section_config)
+    generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT)
+
+    return destination_section_config
+
+
+def convert_legacy_parsed_config(source_config, schema):
+    '''
+    Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its
+    corresponding yaml.comments.CommentedMap representation in preparation for actual serialization
+    to YAML.
+
+    Additionally, use the given schema as a source of helpful comments to include within the
+    returned CommentedMap.
+    '''
+    destination_config = yaml.comments.CommentedMap([
+        (section_name, _convert_section(section_config, schema['map'][section_name]))
+        for section_name, section_config in source_config._asdict().items()
+    ])
+
+    destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
+
+    if source_config.consistency['checks']:
+        destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
+
+    generate.add_comments_to_configuration(destination_config, schema)
+
+    return destination_config
diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py
new file mode 100644
index 000000000..cf4861b85
--- /dev/null
+++ b/borgmatic/config/generate.py
@@ -0,0 +1,90 @@
+from collections import OrderedDict
+
+from ruamel import yaml
+
+
+INDENT = 4
+
+
+def write_configuration(config_filename, config):
+    '''
+    Given a target config filename and a config data structure of nested OrderedDicts, write out the
+    config to file as YAML.
+    '''
+    with open(config_filename, 'w') as config_file:
+        config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
+
+
+def _insert_newline_before_comment(config, field_name):
+    '''
+    Using some ruamel.yaml black magic, insert a blank line in the config right befor the given
+    field and its comments.
+    '''
+    config.ca.items[field_name][1].insert(
+        0,
+        yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None),
+    )
+
+
+def add_comments_to_configuration(config, schema, indent=0):
+    '''
+    Using descriptions from a schema as a source, add those descriptions as comments to the given
+    config before each field. This function only adds comments for the top-most config map level.
+    Indent the comment the given number of characters.
+    '''
+    for index, field_name in enumerate(config.keys()):
+        field_schema = schema['map'].get(field_name, {})
+        description = field_schema.get('desc')
+
+        # No description to use? Skip it.
+        if not schema or not description:
+            continue
+
+        config.yaml_set_comment_before_after_key(
+            key=field_name,
+            before=description,
+            indent=indent,
+        )
+        if index > 0:
+            _insert_newline_before_comment(config, field_name)
+
+
+def _section_schema_to_sample_configuration(section_schema):
+    '''
+    Given the schema for a particular config section, generate and return sample config for that
+    section. Include comments for each field based on the schema "desc" description.
+    '''
+    section_config = yaml.comments.CommentedMap([
+        (field_name, field_schema['example'])
+        for field_name, field_schema in section_schema['map'].items()
+    ])
+
+    add_comments_to_configuration(section_config, section_schema, indent=INDENT)
+
+    return section_config
+
+
+def _schema_to_sample_configuration(schema):
+    '''
+    Given a loaded configuration schema, generate and return sample config for it. Include comments
+    for each section based on the schema "desc" description.
+    '''
+    config = yaml.comments.CommentedMap([
+        (section_name, _section_schema_to_sample_configuration(section_schema))
+        for section_name, section_schema in schema['map'].items()
+    ])
+
+    add_comments_to_configuration(config, schema)
+
+    return config
+
+
+def generate_sample_configuration(config_filename, schema_filename):
+    '''
+    Given a target config filename and the path to a schema filename in pykwalify YAML schema
+    format, write out a sample configuration file based on that schema.
+    '''
+    schema = yaml.round_trip_load(open(schema_filename))
+    config = _schema_to_sample_configuration(schema)
+
+    write_configuration(config_filename, config)
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index 83c50bb19..2cc2f4e9e 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -1,48 +1,110 @@
+name: Borgmatic configuration file schema
 map:
     location:
+        desc: |
+            Where to look for files to backup, and where to store those backups. See
+            https://borgbackup.readthedocs.io/en/stable/quickstart.html and
+            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details.
         required: True
         map:
             source_directories:
                 required: True
                 seq:
                     - type: scalar
+                desc: List of source directories to backup. Globs are expanded.
+                example:
+                    - /home
+                    - /etc
+                    - /var/log/syslog*
             one_file_system:
                 type: bool
+                desc: Stay in same file system (do not cross mount points).
+                example: yes
             remote_path:
                 type: scalar
+                desc: Alternate Borg remote executable. Defaults to "borg".
+                example: borg1
             repository:
                 required: True
                 type: scalar
+                desc: Path to local or remote repository.
+                example: user@backupserver:sourcehostname.borg
     storage:
+        desc: |
+            Repository storage options. See
+            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
+            https://borgbackup.readthedocs.io/en/stable/usage.html#environment-variables for details.
         map:
             encryption_passphrase:
                 type: scalar
+                desc: |
+                    Passphrase to unlock the encryption key with. Only use on repositories that were
+                    initialized with passphrase/repokey encryption. Quote the value if it contains
+                    punctuation, so it parses correctly. And backslash any quote or backslash
+                    literals as well.
+                example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
             compression:
                 type: scalar
+                desc: |
+                    Type of compression to use when creating archives. See
+                    https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
+                    Defaults to no compression.
+                example: lz4
             umask:
                 type: scalar
+                desc: Umask to be used for borg create.
+                example: 0077
     retention:
+        desc: |
+            Retention policy for how many backups to keep in each category. See
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
         map:
             keep_within:
                 type: scalar
+                desc: Keep all archives within this time interval.
+                example: 3H
             keep_hourly:
                 type: int
+                desc: Number of hourly archives to keep.
+                example: 24
             keep_daily:
                 type: int
+                desc: Number of daily archives to keep.
+                example: 7
             keep_weekly:
                 type: int
+                desc: Number of weekly archives to keep.
+                example: 4
             keep_monthly:
                 type: int
+                desc: Number of monthly archives to keep.
+                example: 6
             keep_yearly:
                 type: int
+                desc: Number of yearly archives to keep.
+                example: 1
             prefix:
                 type: scalar
+                desc: When pruning, only consider archive names starting with this prefix.
+                example: sourcehostname
     consistency:
+        desc: |
+            Consistency checks to run after backups. See
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
         map:
             checks:
                 seq:
                     - type: str
                       enum: ['repository', 'archives', 'disabled']
                       unique: True
+                desc: |
+                    List of consistency checks to run: "repository", "archives", or both. Defaults
+                    to both. Set to "disabled" to disable all consistency checks. See
+                    https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
+                example:
+                    - repository
+                    - archives
             check_last:
                 type: int
+                desc: Restrict the number of checked archives to the last n.
+                example: 3
diff --git a/borgmatic/config/yaml.py b/borgmatic/config/validate.py
similarity index 65%
rename from borgmatic/config/yaml.py
rename to borgmatic/config/validate.py
index 4ff01b98d..5dae6f383 100644
--- a/borgmatic/config/yaml.py
+++ b/borgmatic/config/validate.py
@@ -5,7 +5,7 @@ import warnings
 import pkg_resources
 import pykwalify.core
 import pykwalify.errors
-import ruamel.yaml.error
+from ruamel import yaml
 
 
 def schema_filename():
@@ -38,20 +38,18 @@ def parse_configuration(config_filename, schema_filename):
     Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
     have permissions to read the file, or Validation_error if the config does not match the schema.
     '''
-    warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
-    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
-
     try:
-        validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
-    except pykwalify.errors.CoreError as error:
-        if 'do not exists on disk' in str(error):
-            raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
-        if 'Unable to load any data' in str(error):
-            # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
-            # So reach back to the originating exception from ruamel.yaml for something more useful.
-            raise Validation_error(config_filename, (error.__context__,))
-        raise
+        schema = yaml.round_trip_load(open(schema_filename))
+    except yaml.error.YAMLError as error:
+        raise Validation_error(config_filename, (str(error),))
 
+    # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
+    # simply remove all examples before passing the schema to pykwalify.
+    for section_name, section_schema in schema['map'].items():
+        for field_name, field_schema in section_schema['map'].items():
+            field_schema.pop('example')
+
+    validator = pykwalify.core.Core(source_file=config_filename, schema_data=schema)
     parsed_result = validator.validate(raise_exception=False)
 
     if validator.validation_errors:
@@ -73,12 +71,3 @@ def display_validation_error(validation_error):
 
     for error in validation_error.error_messages:
         print(error, file=sys.stderr)
-
-
-# FOR TESTING
-if __name__ == '__main__':
-    try:
-        configuration = parse_configuration('sample/config.yaml', schema_filename())
-        print(configuration)
-    except Validation_error as error:
-        display_validation_error(error)
diff --git a/borgmatic/tests/integration/commands/__init__.py b/borgmatic/tests/integration/commands/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/borgmatic/tests/integration/test_command.py b/borgmatic/tests/integration/commands/test_borgmatic.py
similarity index 97%
rename from borgmatic/tests/integration/test_command.py
rename to borgmatic/tests/integration/commands/test_borgmatic.py
index c1e311c59..69c24686a 100644
--- a/borgmatic/tests/integration/test_command.py
+++ b/borgmatic/tests/integration/commands/test_borgmatic.py
@@ -4,7 +4,7 @@ import sys
 from flexmock import flexmock
 import pytest
 
-from borgmatic import command as module
+from borgmatic.commands import borgmatic as module
 
 
 def test_parse_arguments_with_no_arguments_uses_defaults():
diff --git a/borgmatic/tests/integration/config/test_yaml.py b/borgmatic/tests/integration/config/test_validate.py
similarity index 88%
rename from borgmatic/tests/integration/config/test_yaml.py
rename to borgmatic/tests/integration/config/test_validate.py
index ebd33a369..4769551a2 100644
--- a/borgmatic/tests/integration/config/test_yaml.py
+++ b/borgmatic/tests/integration/config/test_validate.py
@@ -6,7 +6,7 @@ import os
 from flexmock import flexmock
 import pytest
 
-from borgmatic.config import yaml as module
+from borgmatic.config import validate as module
 
 
 def test_schema_filename_returns_plausable_path():
@@ -18,13 +18,13 @@ def test_schema_filename_returns_plausable_path():
 def mock_config_and_schema(config_yaml):
     '''
     Set up mocks for the config config YAML string and the default schema so that pykwalify consumes
-    them when parsing the configuration. This is a little brittle in that it's relying on pykwalify
-    to open() the respective files in a particular order.
+    them when parsing the configuration. This is a little brittle in that it's relying on the code
+    under test to open() the respective files in a particular order.
     '''
-    config_stream = io.StringIO(config_yaml)
     schema_stream = open(module.schema_filename())
+    config_stream = io.StringIO(config_yaml)
     builtins = flexmock(sys.modules['builtins']).should_call('open').mock
-    builtins.should_receive('open').and_return(config_stream).and_return(schema_stream)
+    builtins.should_receive('open').and_return(schema_stream).and_return(config_stream)
     flexmock(os.path).should_receive('exists').and_return(True)
 
 
@@ -87,7 +87,8 @@ def test_parse_configuration_raises_for_missing_config_file():
 
 def test_parse_configuration_raises_for_missing_schema_file():
     mock_config_and_schema('')
-    flexmock(os.path).should_receive('exists').with_args('schema.yaml').and_return(False)
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError)
 
     with pytest.raises(FileNotFoundError):
         module.parse_configuration('config.yaml', 'schema.yaml')
diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py
new file mode 100644
index 000000000..39fc38520
--- /dev/null
+++ b/borgmatic/tests/unit/config/test_convert.py
@@ -0,0 +1,44 @@
+from collections import defaultdict, OrderedDict, namedtuple
+
+from borgmatic.config import convert as module
+
+
+Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))
+
+
+def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
+    source_config = Parsed_config(
+        location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
+        storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
+        retention=OrderedDict([('keep_daily', 7)]),
+        consistency=OrderedDict([('checks', 'repository')]),
+    )
+    schema = {'map': defaultdict(lambda: {'map': {}})}
+
+    destination_config = module.convert_legacy_parsed_config(source_config, schema)
+
+    assert destination_config == OrderedDict([
+        ('location', OrderedDict([('source_directories', ['/home']), ('repository', 'hostname.borg')])),
+        ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])),
+        ('retention', OrderedDict([('keep_daily', 7)])),
+        ('consistency', OrderedDict([('checks', ['repository'])])),
+    ])
+
+
+def test_convert_legacy_parsed_config_splits_space_separated_values():
+    source_config = Parsed_config(
+        location=OrderedDict([('source_directories', '/home /etc')]),
+        storage=OrderedDict(),
+        retention=OrderedDict(),
+        consistency=OrderedDict([('checks', 'repository archives')]),
+    )
+    schema = {'map': defaultdict(lambda: {'map': {}})}
+
+    destination_config = module.convert_legacy_parsed_config(source_config, schema) 
+
+    assert destination_config == OrderedDict([
+        ('location', OrderedDict([('source_directories', ['/home', '/etc'])])),
+        ('storage', OrderedDict()),
+        ('retention', OrderedDict()),
+        ('consistency', OrderedDict([('checks', ['repository', 'archives'])])),
+    ])
diff --git a/sample/config.yaml b/sample/config.yaml
index 6ca27c723..daac2305f 100644
--- a/sample/config.yaml
+++ b/sample/config.yaml
@@ -16,8 +16,10 @@ location:
 
 #storage:
     # Passphrase to unlock the encryption key with. Only use on repositories
-    # that were initialized with passphrase/repokey encryption.
-    #encryption_passphrase: foo
+    # that were initialized with passphrase/repokey encryption. Quote the value
+    # if it contains punctuation so it parses correctly. And backslash any
+    # quote or backslash literals as well.
+    #encryption_passphrase: "foo"
 
     # Type of compression to use when creating archives. See
     # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
diff --git a/setup.py b/setup.py
index 198a2d55f..8ffeaba83 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.0'
+VERSION = '1.1.0.dev0'
 
 
 setup(
@@ -24,7 +24,8 @@ setup(
     packages=find_packages(),
     entry_points={
         'console_scripts': [
-            'borgmatic = borgmatic.command:main',
+            'borgmatic = borgmatic.commands.borgmatic:main',
+            'convert-borgmatic-config = borgmatic.commands.convert_config:main',
         ]
     },
     obsoletes=[
diff --git a/test_requirements.txt b/test_requirements.txt
index 3c5f9e5c9..6bf66e087 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -1,2 +1,5 @@
 flexmock==0.10.2
+pykwalify==1.6.0
 pytest==2.9.1
+pytest-cov==2.5.1
+ruamel.yaml==0.15.18
diff --git a/tox.ini b/tox.ini
index 579a91956..7f6a37540 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,4 +5,4 @@ skipsdist=True
 [testenv]
 usedevelop=True
 deps=-rtest_requirements.txt
-commands = py.test borgmatic []
+commands = py.test --cov=borgmatic borgmatic []

From e50fd04750cdc5175910b31b5da83713175cd00e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 8 Jul 2017 23:01:41 -0700
Subject: [PATCH 125/189] Adding test coverage report. Making tests a little
 less brittle.

---
 borgmatic/config/validate.py                    |  5 +++--
 .../tests/integration/config/test_validate.py   | 17 ++++++++---------
 tox.ini                                         |  2 +-
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py
index 5dae6f383..82b4f93b9 100644
--- a/borgmatic/config/validate.py
+++ b/borgmatic/config/validate.py
@@ -39,6 +39,7 @@ def parse_configuration(config_filename, schema_filename):
     have permissions to read the file, or Validation_error if the config does not match the schema.
     '''
     try:
+        config = yaml.round_trip_load(open(config_filename))
         schema = yaml.round_trip_load(open(schema_filename))
     except yaml.error.YAMLError as error:
         raise Validation_error(config_filename, (str(error),))
@@ -49,7 +50,7 @@ def parse_configuration(config_filename, schema_filename):
         for field_name, field_schema in section_schema['map'].items():
             field_schema.pop('example')
 
-    validator = pykwalify.core.Core(source_file=config_filename, schema_data=schema)
+    validator = pykwalify.core.Core(source_data=config, schema_data=schema)
     parsed_result = validator.validate(raise_exception=False)
 
     if validator.validation_errors:
@@ -58,7 +59,7 @@ def parse_configuration(config_filename, schema_filename):
     return parsed_result
 
 
-def display_validation_error(validation_error):
+def display_validation_error(validation_error):  # pragma: no cover
     '''
     Given a Validation_error, display its error messages to stderr.
     '''
diff --git a/borgmatic/tests/integration/config/test_validate.py b/borgmatic/tests/integration/config/test_validate.py
index 4769551a2..897ea410a 100644
--- a/borgmatic/tests/integration/config/test_validate.py
+++ b/borgmatic/tests/integration/config/test_validate.py
@@ -17,15 +17,14 @@ def test_schema_filename_returns_plausable_path():
 
 def mock_config_and_schema(config_yaml):
     '''
-    Set up mocks for the config config YAML string and the default schema so that pykwalify consumes
-    them when parsing the configuration. This is a little brittle in that it's relying on the code
-    under test to open() the respective files in a particular order.
+    Set up mocks for the config config YAML string and the default schema so that the code under
+    test consumes them when parsing the configuration.
     '''
-    schema_stream = open(module.schema_filename())
     config_stream = io.StringIO(config_yaml)
-    builtins = flexmock(sys.modules['builtins']).should_call('open').mock
-    builtins.should_receive('open').and_return(schema_stream).and_return(config_stream)
-    flexmock(os.path).should_receive('exists').and_return(True)
+    schema_stream = open(module.schema_filename())
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_stream)
+    builtins.should_receive('open').with_args('schema.yaml').and_return(schema_stream)
 
 
 def test_parse_configuration_transforms_file_into_mapping():
@@ -95,9 +94,9 @@ def test_parse_configuration_raises_for_missing_schema_file():
 
 
 def test_parse_configuration_raises_for_syntax_error():
-    mock_config_and_schema('invalid = yaml')
+    mock_config_and_schema('foo:\nbar')
 
-    with pytest.raises(module.Validation_error):
+    with pytest.raises(ValueError):
         module.parse_configuration('config.yaml', 'schema.yaml')
 
 
diff --git a/tox.ini b/tox.ini
index 7f6a37540..159847d76 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,4 +5,4 @@ skipsdist=True
 [testenv]
 usedevelop=True
 deps=-rtest_requirements.txt
-commands = py.test --cov=borgmatic borgmatic []
+commands = py.test --cov-report term-missing:skip-covered --cov=borgmatic borgmatic []

From a16d90ff464a11cd75bff89fac513d3acc647107 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 9 Jul 2017 10:27:34 -0700
Subject: [PATCH 126/189] Adding a "does not raise" test for displaying errors.

---
 borgmatic/config/validate.py                        | 2 +-
 borgmatic/tests/integration/config/test_validate.py | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py
index 82b4f93b9..312517626 100644
--- a/borgmatic/config/validate.py
+++ b/borgmatic/config/validate.py
@@ -59,7 +59,7 @@ def parse_configuration(config_filename, schema_filename):
     return parsed_result
 
 
-def display_validation_error(validation_error):  # pragma: no cover
+def display_validation_error(validation_error):
     '''
     Given a Validation_error, display its error messages to stderr.
     '''
diff --git a/borgmatic/tests/integration/config/test_validate.py b/borgmatic/tests/integration/config/test_validate.py
index 897ea410a..90e223d13 100644
--- a/borgmatic/tests/integration/config/test_validate.py
+++ b/borgmatic/tests/integration/config/test_validate.py
@@ -111,3 +111,10 @@ def test_parse_configuration_raises_for_validation_error():
 
     with pytest.raises(module.Validation_error):
         module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_display_validation_error_does_not_raise():
+    flexmock(sys.modules['builtins']).should_receive('print')
+    error = module.Validation_error('config.yaml', ('oops', 'uh oh'))
+
+    module.display_validation_error(error)

From 1bcb2a8be402508fcccad6925b14b4bd8da77c9f Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 9 Jul 2017 11:41:55 -0700
Subject: [PATCH 127/189] More test coverage, and simplification of config
 generation.

---
 .hgignore                                     |  1 +
 borgmatic/commands/generate_config.py         | 39 ++++++++++
 borgmatic/config/generate.py                  | 72 +++++++++----------
 .../tests/integration/config/test_generate.py | 43 +++++++++++
 borgmatic/tests/unit/config/test_generate.py  | 49 +++++++++++++
 setup.py                                      |  1 +
 6 files changed, 165 insertions(+), 40 deletions(-)
 create mode 100644 borgmatic/commands/generate_config.py
 create mode 100644 borgmatic/tests/integration/config/test_generate.py
 create mode 100644 borgmatic/tests/unit/config/test_generate.py

diff --git a/.hgignore b/.hgignore
index f41ba9356..fd7c96da7 100644
--- a/.hgignore
+++ b/.hgignore
@@ -3,6 +3,7 @@ syntax: glob
 *.pyc
 *.swp
 .cache
+.coverage
 .tox
 build
 dist
diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py
new file mode 100644
index 000000000..f326d6ae9
--- /dev/null
+++ b/borgmatic/commands/generate_config.py
@@ -0,0 +1,39 @@
+from __future__ import print_function
+from argparse import ArgumentParser
+import os
+from subprocess import CalledProcessError
+import sys
+
+from ruamel import yaml
+
+from borgmatic import borg
+from borgmatic.config import convert, generate, validate
+
+
+DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
+    parser.add_argument(
+        '-d', '--destination',
+        dest='destination_filename',
+        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
+        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():
+    try:
+        args = parse_arguments(*sys.argv[1:])
+
+        generate.generate_sample_configuration(args.destination_filename, validate.schema_filename())
+    except (ValueError, OSError) as error:
+        print(error, file=sys.stderr)
+        sys.exit(1)
diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py
index cf4861b85..66d6c44b0 100644
--- a/borgmatic/config/generate.py
+++ b/borgmatic/config/generate.py
@@ -6,15 +6,6 @@ from ruamel import yaml
 INDENT = 4
 
 
-def write_configuration(config_filename, config):
-    '''
-    Given a target config filename and a config data structure of nested OrderedDicts, write out the
-    config to file as YAML.
-    '''
-    with open(config_filename, 'w') as config_file:
-        config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
-
-
 def _insert_newline_before_comment(config, field_name):
     '''
     Using some ruamel.yaml black magic, insert a blank line in the config right befor the given
@@ -26,6 +17,37 @@ def _insert_newline_before_comment(config, field_name):
     )
 
 
+def _schema_to_sample_configuration(schema, level=0):
+    '''
+    Given a loaded configuration schema, generate and return sample config for it. Include comments
+    for each section based on the schema "desc" description.
+    '''
+    example = schema.get('example')
+    if example:
+        return example
+
+    config = yaml.comments.CommentedMap([
+        (
+            section_name,
+            _schema_to_sample_configuration(section_schema, level + 1),
+        )
+        for section_name, section_schema in schema['map'].items()
+    ])
+
+    add_comments_to_configuration(config, schema, indent=(level * INDENT))
+
+    return config
+
+
+def write_configuration(config_filename, config):
+    '''
+    Given a target config filename and a config data structure of nested OrderedDicts, write out the
+    config to file as YAML.
+    '''
+    with open(config_filename, 'w') as config_file:
+        config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
+
+
 def add_comments_to_configuration(config, schema, indent=0):
     '''
     Using descriptions from a schema as a source, add those descriptions as comments to the given
@@ -37,7 +59,7 @@ def add_comments_to_configuration(config, schema, indent=0):
         description = field_schema.get('desc')
 
         # No description to use? Skip it.
-        if not schema or not description:
+        if not field_schema or not description:
             continue
 
         config.yaml_set_comment_before_after_key(
@@ -49,36 +71,6 @@ def add_comments_to_configuration(config, schema, indent=0):
             _insert_newline_before_comment(config, field_name)
 
 
-def _section_schema_to_sample_configuration(section_schema):
-    '''
-    Given the schema for a particular config section, generate and return sample config for that
-    section. Include comments for each field based on the schema "desc" description.
-    '''
-    section_config = yaml.comments.CommentedMap([
-        (field_name, field_schema['example'])
-        for field_name, field_schema in section_schema['map'].items()
-    ])
-
-    add_comments_to_configuration(section_config, section_schema, indent=INDENT)
-
-    return section_config
-
-
-def _schema_to_sample_configuration(schema):
-    '''
-    Given a loaded configuration schema, generate and return sample config for it. Include comments
-    for each section based on the schema "desc" description.
-    '''
-    config = yaml.comments.CommentedMap([
-        (section_name, _section_schema_to_sample_configuration(section_schema))
-        for section_name, section_schema in schema['map'].items()
-    ])
-
-    add_comments_to_configuration(config, schema)
-
-    return config
-
-
 def generate_sample_configuration(config_filename, schema_filename):
     '''
     Given a target config filename and the path to a schema filename in pykwalify YAML schema
diff --git a/borgmatic/tests/integration/config/test_generate.py b/borgmatic/tests/integration/config/test_generate.py
new file mode 100644
index 000000000..89d846717
--- /dev/null
+++ b/borgmatic/tests/integration/config/test_generate.py
@@ -0,0 +1,43 @@
+from io import StringIO
+import sys
+
+from flexmock import flexmock
+
+from borgmatic.config import generate as module
+
+
+def test_insert_newline_before_comment_does_not_raise():
+    field_name = 'foo'
+    config = module.yaml.comments.CommentedMap([(field_name, 33)])
+    config.yaml_set_comment_before_after_key(key=field_name, before='Comment',)
+
+    module._insert_newline_before_comment(config, field_name)
+
+
+def test_write_configuration_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').and_return(StringIO())
+
+    module.write_configuration('config.yaml', {})
+
+
+def test_add_comments_to_configuration_does_not_raise():
+    # Ensure that it can deal with fields both in the schema and missing from the schema.
+    config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
+    schema = {
+        'map': {
+            'foo': {'desc': 'Foo'},
+            'bar': {'desc': 'Bar'},
+        }
+    }
+
+    module.add_comments_to_configuration(config, schema)
+
+
+def test_generate_sample_configuration_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_return('')
+    flexmock(module).should_receive('write_configuration')
+    flexmock(module).should_receive('_schema_to_sample_configuration')
+
+    module.generate_sample_configuration('config.yaml', 'schema.yaml')
diff --git a/borgmatic/tests/unit/config/test_generate.py b/borgmatic/tests/unit/config/test_generate.py
new file mode 100644
index 000000000..3536f239b
--- /dev/null
+++ b/borgmatic/tests/unit/config/test_generate.py
@@ -0,0 +1,49 @@
+from collections import OrderedDict
+
+from flexmock import flexmock
+
+from borgmatic.config import generate as module
+
+
+def test_schema_to_sample_configuration_generates_config_with_examples():
+    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
+    flexmock(module).should_receive('add_comments_to_configuration')
+    schema = {
+        'map': OrderedDict([
+            (
+                'section1', {
+                    'map': {
+                        'field1': OrderedDict([
+                            ('example', 'Example 1')
+                        ]),
+                    },
+                },
+            ),
+            (
+                'section2', {
+                    'map': OrderedDict([
+                        ('field2', {'example': 'Example 2'}),
+                        ('field3', {'example': 'Example 3'}),
+                    ]),
+                }
+            ),
+        ])
+    }
+
+    config = module._schema_to_sample_configuration(schema)
+
+    assert config == OrderedDict([
+        (
+            'section1',
+            OrderedDict([
+                ('field1', 'Example 1'),
+            ]),
+        ),
+        (
+            'section2',
+            OrderedDict([
+                ('field2', 'Example 2'),
+                ('field3', 'Example 3'),
+            ]),
+        )
+    ])
diff --git a/setup.py b/setup.py
index 8ffeaba83..6e9d53a3a 100644
--- a/setup.py
+++ b/setup.py
@@ -26,6 +26,7 @@ setup(
         'console_scripts': [
             'borgmatic = borgmatic.commands.borgmatic:main',
             'convert-borgmatic-config = borgmatic.commands.convert_config:main',
+            'generate-borgmatic-config = borgmatic.commands.generate_config:main',
         ]
     },
     obsoletes=[

From dc9b075d5ab122f6527de533be6fc3277328bf6e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 9 Jul 2017 11:48:24 -0700
Subject: [PATCH 128/189] Rename convert-borgmatic-config to
 upgrade-borgmatic-config.

---
 NEWS     | 2 +-
 setup.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 3d16ba536..3209b41ff 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,6 @@
 1.1.0.dev0
 
- * Switched config file format to YAML. Run convert-borgmatic-config to upgrade.
+ * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade.
  * Dropped Python 2 support. Now Python 3 only.
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.
diff --git a/setup.py b/setup.py
index 6e9d53a3a..8f8da5c24 100644
--- a/setup.py
+++ b/setup.py
@@ -25,7 +25,7 @@ setup(
     entry_points={
         'console_scripts': [
             'borgmatic = borgmatic.commands.borgmatic:main',
-            'convert-borgmatic-config = borgmatic.commands.convert_config:main',
+            'upgrade-borgmatic-config = borgmatic.commands.convert_config:main',
             'generate-borgmatic-config = borgmatic.commands.generate_config:main',
         ]
     },

From d4ae7814a03d9caa612a84ae3eb6a6f378b18419 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 9 Jul 2017 11:49:51 -0700
Subject: [PATCH 129/189] Adding TODO about a helpful notice about legacy
 config.

---
 borgmatic/commands/borgmatic.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index c817976a8..4c2e9bab9 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -41,6 +41,9 @@ def parse_arguments(*arguments):
 
 def main():
     try:
+        # TODO: Detect whether only legacy config is present. If so, inform the user about how to
+        # upgrade, then exet.
+
         args = parse_arguments(*sys.argv[1:])
         config = parse_configuration(args.config_filename, schema_filename())
         repository = config.location['repository']

From d49be195448cae5e493113c8f851fe4ff1582c0f Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 9 Jul 2017 16:18:10 -0700
Subject: [PATCH 130/189] Add a version to the schema, because inevitably I'll
 want to revise the schema.

---
 borgmatic/config/schema.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index 2cc2f4e9e..347dfee80 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -1,4 +1,5 @@
 name: Borgmatic configuration file schema
+version: 1
 map:
     location:
         desc: |

From 17c87f8758ce86be264964af9b69dd41538d1ff4 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 9 Jul 2017 17:03:45 -0700
Subject: [PATCH 131/189] Completed test coverage of commands (except for
 main()s).

---
 borgmatic/commands/borgmatic.py                    |  2 +-
 borgmatic/commands/convert_config.py               |  3 ++-
 borgmatic/commands/generate_config.py              |  2 +-
 .../integration/commands/test_convert_config.py    | 14 ++++++++++++++
 .../integration/commands/test_generate_config.py   | 12 ++++++++++++
 5 files changed, 30 insertions(+), 3 deletions(-)
 create mode 100644 borgmatic/tests/integration/commands/test_convert_config.py
 create mode 100644 borgmatic/tests/integration/commands/test_generate_config.py

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 4c2e9bab9..1fcc582d0 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -39,7 +39,7 @@ def parse_arguments(*arguments):
     return parser.parse_args(arguments)
 
 
-def main():
+def main():  # pragma: no cover
     try:
         # TODO: Detect whether only legacy config is present. If so, inform the user about how to
         # upgrade, then exet.
diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py
index 8f645c0aa..77735bf7f 100644
--- a/borgmatic/commands/convert_config.py
+++ b/borgmatic/commands/convert_config.py
@@ -11,6 +11,7 @@ from borgmatic.config import convert, generate, legacy, validate
 
 
 DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
+# TODO: Fold excludes into the YAML config file.
 DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 
@@ -37,7 +38,7 @@ def parse_arguments(*arguments):
     return parser.parse_args(arguments)
 
 
-def main():
+def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
         source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py
index f326d6ae9..0f0474086 100644
--- a/borgmatic/commands/generate_config.py
+++ b/borgmatic/commands/generate_config.py
@@ -29,7 +29,7 @@ def parse_arguments(*arguments):
     return parser.parse_args(arguments)
 
 
-def main():
+def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
 
diff --git a/borgmatic/tests/integration/commands/test_convert_config.py b/borgmatic/tests/integration/commands/test_convert_config.py
new file mode 100644
index 000000000..b2bc32bd8
--- /dev/null
+++ b/borgmatic/tests/integration/commands/test_convert_config.py
@@ -0,0 +1,14 @@
+from borgmatic.commands import convert_config as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    parser = module.parse_arguments()
+
+    assert parser.source_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
+    assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+
+def test_parse_arguments_with_filename_arguments_overrides_defaults():
+    parser = module.parse_arguments('--source', 'config', '--destination', 'config.yaml')
+
+    assert parser.source_filename == 'config'
+    assert parser.destination_filename == 'config.yaml'
diff --git a/borgmatic/tests/integration/commands/test_generate_config.py b/borgmatic/tests/integration/commands/test_generate_config.py
new file mode 100644
index 000000000..b9b4db082
--- /dev/null
+++ b/borgmatic/tests/integration/commands/test_generate_config.py
@@ -0,0 +1,12 @@
+from borgmatic.commands import generate_config as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    parser = module.parse_arguments()
+
+    assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+
+def test_parse_arguments_with_filename_argument_overrides_defaults():
+    parser = module.parse_arguments('--destination', 'config.yaml')
+
+    assert parser.destination_filename == 'config.yaml'

From fea97b514950fb96bf6dd716e7e0344f81bed041 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 09:43:25 -0700
Subject: [PATCH 132/189] Merge excludes into config file format.

---
 borgmatic/commands/convert_config.py          | 33 ++++++++++----
 borgmatic/config/convert.py                   | 19 +++++---
 borgmatic/config/schema.yaml                  | 12 ++++++
 .../integration/commands/test_borgmatic.py    | 20 +++------
 .../commands/test_convert_config.py           | 43 ++++++++++++++++---
 5 files changed, 95 insertions(+), 32 deletions(-)

diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py
index 77735bf7f..2b99863b2 100644
--- a/borgmatic/commands/convert_config.py
+++ b/borgmatic/commands/convert_config.py
@@ -11,7 +11,6 @@ from borgmatic.config import convert, generate, legacy, validate
 
 
 DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
-# TODO: Fold excludes into the YAML config file.
 DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 
@@ -21,16 +20,27 @@ def parse_arguments(*arguments):
     Given command-line arguments with which this script was invoked, parse the arguments and return
     them as an ArgumentParser instance.
     '''
-    parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.')
+    parser = ArgumentParser(
+        description='''
+            Convert legacy INI-style borgmatic configuration and excludes files to a single YAML
+            configuration file. Note that this replaces any comments from the source files.
+        '''
+    )
     parser.add_argument(
-        '-s', '--source',
-        dest='source_filename',
+        '-s', '--source-config',
+        dest='source_config_filename',
         default=DEFAULT_SOURCE_CONFIG_FILENAME,
         help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
     )
     parser.add_argument(
-        '-d', '--destination',
-        dest='destination_filename',
+        '-e', '--source-excludes',
+        dest='source_excludes_filename',
+        default=DEFAULT_SOURCE_EXCLUDES_FILENAME if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) else None,
+        help='Excludes filename',
+    )
+    parser.add_argument(
+        '-d', '--destination-config',
+        dest='destination_config_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
         help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
     )
@@ -41,12 +51,17 @@ def parse_arguments(*arguments):
 def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
-        source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
         schema = yaml.round_trip_load(open(validate.schema_filename()).read())
+        source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT)
+        source_excludes = (
+            open(args.source_excludes_filename).read().splitlines()
+            if args.source_excludes_filename
+            else []
+        )
 
-        destination_config = convert.convert_legacy_parsed_config(source_config, schema)
+        destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema)
 
-        generate.write_configuration(args.destination_filename, destination_config)
+        generate.write_configuration(args.destination_config_filename, destination_config)
 
         # TODO: As a backstop, check that the written config can actually be read and parsed, and
         # that it matches the destination config data structure that was written.
diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py
index 72b9ba314..11618e958 100644
--- a/borgmatic/config/convert.py
+++ b/borgmatic/config/convert.py
@@ -12,16 +12,15 @@ def _convert_section(source_section_config, section_schema):
     returned CommentedMap.
     '''
     destination_section_config = yaml.comments.CommentedMap(source_section_config)
-    generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT)
 
     return destination_section_config
 
 
-def convert_legacy_parsed_config(source_config, schema):
+def convert_legacy_parsed_config(source_config, source_excludes, schema):
     '''
-    Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its
-    corresponding yaml.comments.CommentedMap representation in preparation for actual serialization
-    to YAML.
+    Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude
+    patterns, convert them to a corresponding yaml.comments.CommentedMap representation in
+    preparation for serialization to a single YAML config file.
 
     Additionally, use the given schema as a source of helpful comments to include within the
     returned CommentedMap.
@@ -31,11 +30,21 @@ def convert_legacy_parsed_config(source_config, schema):
         for section_name, section_config in source_config._asdict().items()
     ])
 
+    # Split space-seperated values into actual lists, and merge in excludes.
     destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
+    destination_config['location']['exclude_patterns'] = source_excludes
 
     if source_config.consistency['checks']:
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
 
+    # Add comments to each section, and then add comments to the fields in each section.
     generate.add_comments_to_configuration(destination_config, schema)
 
+    for section_name, section_config in destination_config.items():
+        generate.add_comments_to_configuration(
+            section_config,
+            schema['map'][section_name],
+            indent=generate.INDENT,
+        )
+
     return destination_config
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index 347dfee80..97f13d98a 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -30,6 +30,18 @@ map:
                 type: scalar
                 desc: Path to local or remote repository.
                 example: user@backupserver:sourcehostname.borg
+            exclude_patterns:
+                seq:
+                    - type: scalar
+                desc: |
+                    Exclude patterns. Any paths matching these patterns are excluded from backups.
+                    Globs are expanded. See
+                    https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns for
+                    details.
+                example:
+                    - '*.pyc'
+                    - /home/*/.cache
+                    - /etc/ssl
     storage:
         desc: |
             Repository storage options. See
diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py
index 69c24686a..12afeedf0 100644
--- a/borgmatic/tests/integration/commands/test_borgmatic.py
+++ b/borgmatic/tests/integration/commands/test_borgmatic.py
@@ -1,5 +1,4 @@
 import os
-import sys
 
 from flexmock import flexmock
 import pytest
@@ -14,7 +13,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
     assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
-    assert parser.verbosity == None
+    assert parser.verbosity is None
 
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
@@ -24,7 +23,7 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
 
     assert parser.config_filename == 'myconfig'
     assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity == None
+    assert parser.verbosity is None
 
 
 def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
@@ -33,8 +32,8 @@ def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_non
     parser = module.parse_arguments()
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == None
-    assert parser.verbosity == None
+    assert parser.excludes_filename is None
+    assert parser.verbosity is None
 
 
 def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
@@ -44,7 +43,7 @@ def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
     assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity == None
+    assert parser.verbosity is None
 
 
 def test_parse_arguments_with_verbosity_flag_overrides_default():
@@ -59,11 +58,6 @@ def test_parse_arguments_with_verbosity_flag_overrides_default():
 
 def test_parse_arguments_with_invalid_arguments_exits():
     flexmock(os.path).should_receive('exists').and_return(True)
-    original_stderr = sys.stderr
-    sys.stderr = sys.stdout
 
-    try:
-        with pytest.raises(SystemExit):
-            module.parse_arguments('--posix-me-harder')
-    finally:
-        sys.stderr = original_stderr
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--posix-me-harder')
diff --git a/borgmatic/tests/integration/commands/test_convert_config.py b/borgmatic/tests/integration/commands/test_convert_config.py
index b2bc32bd8..e126c1201 100644
--- a/borgmatic/tests/integration/commands/test_convert_config.py
+++ b/borgmatic/tests/integration/commands/test_convert_config.py
@@ -1,14 +1,47 @@
+import os
+
+from flexmock import flexmock
+import pytest
+
 from borgmatic.commands import convert_config as module
 
 
 def test_parse_arguments_with_no_arguments_uses_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
     parser = module.parse_arguments()
 
-    assert parser.source_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
-    assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+    assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
+    assert parser.source_excludes_filename == module.DEFAULT_SOURCE_EXCLUDES_FILENAME
+    assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
-    parser = module.parse_arguments('--source', 'config', '--destination', 'config.yaml')
+    flexmock(os.path).should_receive('exists').and_return(True)
 
-    assert parser.source_filename == 'config'
-    assert parser.destination_filename == 'config.yaml'
+    parser = module.parse_arguments(
+        '--source-config', 'config',
+        '--source-excludes', 'excludes',
+        '--destination-config', 'config.yaml',
+    )
+
+    assert parser.source_config_filename == 'config'
+    assert parser.source_excludes_filename == 'excludes'
+    assert parser.destination_config_filename == 'config.yaml'
+
+
+def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
+    flexmock(os.path).should_receive('exists').and_return(False)
+
+    parser = module.parse_arguments()
+
+    assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
+    assert parser.source_excludes_filename is None
+    assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--posix-me-harder')

From 338b80903cd5f5a84913f48af8648d09e136a099 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 10:09:06 -0700
Subject: [PATCH 133/189] Fixing tests broken by excludes merging.

---
 borgmatic/tests/unit/config/test_convert.py | 27 ++++++++++++++++++---
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py
index 39fc38520..a88ed2f0c 100644
--- a/borgmatic/tests/unit/config/test_convert.py
+++ b/borgmatic/tests/unit/config/test_convert.py
@@ -1,5 +1,7 @@
 from collections import defaultdict, OrderedDict, namedtuple
 
+from flexmock import flexmock
+
 from borgmatic.config import convert as module
 
 
@@ -7,18 +9,27 @@ Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention',
 
 
 def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
+    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
     source_config = Parsed_config(
         location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
         storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
         retention=OrderedDict([('keep_daily', 7)]),
         consistency=OrderedDict([('checks', 'repository')]),
     )
+    source_excludes = ['/var']
     schema = {'map': defaultdict(lambda: {'map': {}})}
 
-    destination_config = module.convert_legacy_parsed_config(source_config, schema)
+    destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema)
 
     assert destination_config == OrderedDict([
-        ('location', OrderedDict([('source_directories', ['/home']), ('repository', 'hostname.borg')])),
+        (
+            'location',
+            OrderedDict([
+                ('source_directories', ['/home']),
+                ('repository', 'hostname.borg'),
+                ('exclude_patterns', ['/var']),
+            ]),
+        ),
         ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])),
         ('retention', OrderedDict([('keep_daily', 7)])),
         ('consistency', OrderedDict([('checks', ['repository'])])),
@@ -26,18 +37,26 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
 
 
 def test_convert_legacy_parsed_config_splits_space_separated_values():
+    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
     source_config = Parsed_config(
         location=OrderedDict([('source_directories', '/home /etc')]),
         storage=OrderedDict(),
         retention=OrderedDict(),
         consistency=OrderedDict([('checks', 'repository archives')]),
     )
+    source_excludes = ['/var']
     schema = {'map': defaultdict(lambda: {'map': {}})}
 
-    destination_config = module.convert_legacy_parsed_config(source_config, schema) 
+    destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema) 
 
     assert destination_config == OrderedDict([
-        ('location', OrderedDict([('source_directories', ['/home', '/etc'])])),
+        (
+            'location',
+            OrderedDict([
+                ('source_directories', ['/home', '/etc']),
+                ('exclude_patterns', ['/var']),
+            ]),
+        ),
         ('storage', OrderedDict()),
         ('retention', OrderedDict()),
         ('consistency', OrderedDict([('checks', ['repository', 'archives'])])),

From 618e56b2a5f9b2cb381ea805e811cf1f8d76b1f8 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 10:13:57 -0700
Subject: [PATCH 134/189] Display result of config upgrade.

---
 borgmatic/commands/convert_config.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py
index 2b99863b2..eb97ec40f 100644
--- a/borgmatic/commands/convert_config.py
+++ b/borgmatic/commands/convert_config.py
@@ -3,6 +3,7 @@ from argparse import ArgumentParser
 import os
 from subprocess import CalledProcessError
 import sys
+import textwrap
 
 from ruamel import yaml
 
@@ -48,6 +49,30 @@ def parse_arguments(*arguments):
     return parser.parse_args(arguments)
 
 
+TEXT_WRAP_CHARACTERS = 80
+
+
+def display_result(args):  # pragma: no cover
+    result_lines = textwrap.wrap(
+        'Your borgmatic configuration has been upgraded. Please review the result in {}.'.format(
+            args.destination_config_filename
+        ),
+        TEXT_WRAP_CHARACTERS,
+    )
+    
+    delete_lines = textwrap.wrap(
+        'Once you are satisfied, you can safely delete {}{}.'.format(
+            args.source_config_filename,
+            ' and {}'.format(args.source_excludes_filename) if args.source_excludes_filename else '',
+        ),
+        TEXT_WRAP_CHARACTERS,
+    )
+
+    print('\n'.join(result_lines))
+    print()
+    print('\n'.join(delete_lines))
+
+
 def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
@@ -65,6 +90,8 @@ def main():  # pragma: no cover
 
         # TODO: As a backstop, check that the written config can actually be read and parsed, and
         # that it matches the destination config data structure that was written.
+
+        display_result(args)
     except (ValueError, OSError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

From 5ff016238e0c15691221ccd5f330b09b0e5df8ee Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 10:37:11 -0700
Subject: [PATCH 135/189] Don't overwrite config files. And retain file
 permissions when upgrading config.

---
 borgmatic/commands/convert_config.py                |  2 ++
 borgmatic/config/generate.py                        |  4 ++++
 borgmatic/tests/integration/config/test_generate.py | 12 ++++++++++++
 3 files changed, 18 insertions(+)

diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py
index eb97ec40f..eb0bfa876 100644
--- a/borgmatic/commands/convert_config.py
+++ b/borgmatic/commands/convert_config.py
@@ -78,6 +78,7 @@ def main():  # pragma: no cover
         args = parse_arguments(*sys.argv[1:])
         schema = yaml.round_trip_load(open(validate.schema_filename()).read())
         source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT)
+        source_config_file_mode = os.stat(args.source_config_filename).st_mode
         source_excludes = (
             open(args.source_excludes_filename).read().splitlines()
             if args.source_excludes_filename
@@ -87,6 +88,7 @@ def main():  # pragma: no cover
         destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema)
 
         generate.write_configuration(args.destination_config_filename, destination_config)
+        os.chmod(args.destination_config_filename, source_config_file_mode)
 
         # TODO: As a backstop, check that the written config can actually be read and parsed, and
         # that it matches the destination config data structure that was written.
diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py
index 66d6c44b0..8b552d5b7 100644
--- a/borgmatic/config/generate.py
+++ b/borgmatic/config/generate.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import os
 
 from ruamel import yaml
 
@@ -44,6 +45,9 @@ def write_configuration(config_filename, config):
     Given a target config filename and a config data structure of nested OrderedDicts, write out the
     config to file as YAML.
     '''
+    if os.path.exists(config_filename):
+        raise FileExistsError('{} already exists. Aborting.'.format(config_filename))
+
     with open(config_filename, 'w') as config_file:
         config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
 
diff --git a/borgmatic/tests/integration/config/test_generate.py b/borgmatic/tests/integration/config/test_generate.py
index 89d846717..078367d35 100644
--- a/borgmatic/tests/integration/config/test_generate.py
+++ b/borgmatic/tests/integration/config/test_generate.py
@@ -1,7 +1,9 @@
 from io import StringIO
+import os
 import sys
 
 from flexmock import flexmock
+import pytest
 
 from borgmatic.config import generate as module
 
@@ -15,12 +17,22 @@ def test_insert_newline_before_comment_does_not_raise():
 
 
 def test_write_configuration_does_not_raise():
+    flexmock(os.path).should_receive('exists').and_return(False)
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').and_return(StringIO())
 
     module.write_configuration('config.yaml', {})
 
 
+def test_write_configuration_with_already_existing_file_raises():
+    flexmock(os.path).should_receive('exists').and_return(True)
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').and_return(StringIO())
+
+    with pytest.raises(FileExistsError):
+        module.write_configuration('config.yaml', {})
+
+
 def test_add_comments_to_configuration_does_not_raise():
     # Ensure that it can deal with fields both in the schema and missing from the schema.
     config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])

From ff28be77249bc28ba28006c8313fb470166a3d1b Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 11:06:28 -0700
Subject: [PATCH 136/189] Documentation updates based on the new YAML
 configuration.

---
 README.md                             | 54 +++++++++++++++++++-------
 borgmatic/commands/generate_config.py |  5 +++
 borgmatic/config/schema.yaml          | 11 +++---
 sample/config                         | 48 -----------------------
 sample/config.yaml                    | 56 ---------------------------
 sample/excludes                       |  3 --
 6 files changed, 51 insertions(+), 126 deletions(-)
 delete mode 100644 sample/config
 delete mode 100644 sample/config.yaml
 delete mode 100644 sample/excludes

diff --git a/README.md b/README.md
index 5e69283ae..b1225c346 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,10 @@ location:
     # Path to local or remote repository.
     repository: user@backupserver:sourcehostname.borg
 
+    # Any paths matching these patterns are excluded from backups.
+    exclude_patterns:
+        - /home/*/.cache
+
 retention:
     # Retention policy for how many backups to keep in each category.
     keep_daily: 7
@@ -37,16 +41,13 @@ consistency:
         - archives
 ```
 
-Additionally, exclude patterns can be specified in a separate excludes config
-file, one pattern per line.
-
 borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
 available](https://torsion.org/hg/borgmatic). It's also mirrored on
 [GitHub](https://github.com/witten/borgmatic) and
 [BitBucket](https://bitbucket.org/dhelfman/borgmatic) for convenience.
 
 
-## Setup
+## Installation
 
 To get up and running, follow the [Borg Quick
 Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
@@ -66,19 +67,45 @@ To install borgmatic, run the following command to download and install it:
 Make sure you're using Python 3, as borgmatic does not support Python 2. (You
 may have to use "pip3" instead of "pip".)
 
-Then, download a [sample config
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config.yaml) and a
-[sample excludes
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/excludes). From the
-directory where you downloaded them:
+Then, generate a sample configuration file:
 
-    sudo mkdir /etc/borgmatic/
-    sudo mv config.yaml excludes /etc/borgmatic/
+    sudo generate-borgmatic-config
 
-Lastly, modify the /etc files with your desired configuration.
+This generates a sample configuration file at /etc/borgmatic/config.yaml (by
+default). You should edit the file to suit your needs, as the values are just
+representative. All fields are optional except where indicated, so feel free
+to remove anything you don't need.
 
 
-## Upgrading from atticmatic
+## Upgrading
+
+In general, all you should need to do to upgrade borgmatic is run the
+following:
+
+    sudo pip install --upgrade borgmatic
+
+However, see below about special cases.
+
+### Upgrading from borgmatic 1.0.x
+
+borgmatic changed its configuration file format in version 1.1.0 from
+INI-style to YAML. This better supports validation, and has a more natural way
+to express lists of values. To upgrade your existing configuration, first
+upgrade to the new version of borgmatic:
+
+    sudo pip install --upgrade borgmatic
+
+Then, run:
+
+    sudo upgrade-borgmatic-configuration
+
+That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
+(by default) using the values from both your existing configuration and
+excludes files. The new version of borgmatic will consume the YAML
+configuration file instead of the old one.
+
+
+### Upgrading from atticmatic
 
 You can ignore this section if you're not an atticmatic user (the former name
 of borgmatic).
@@ -98,6 +125,7 @@ from atticmatic to borgmatic. Simply run the following commands:
 That's it! borgmatic will continue using your /etc/borgmatic configuration
 files.
 
+
 ## Usage
 
 You can run borgmatic and start a backup simply by invoking it without
diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py
index 0f0474086..ba88ddf8c 100644
--- a/borgmatic/commands/generate_config.py
+++ b/borgmatic/commands/generate_config.py
@@ -34,6 +34,11 @@ def main():  # pragma: no cover
         args = parse_arguments(*sys.argv[1:])
 
         generate.generate_sample_configuration(args.destination_filename, validate.schema_filename())
+
+        print('Generated a sample configuration file at {}.'.format(args.destination_filename))
+        print()
+        print('Please edit the file to suit your needs. The values are just representative.')
+        print('All fields are optional except where indicated.')
     except (ValueError, OSError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index 97f13d98a..dfd0c55bb 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -12,7 +12,7 @@ map:
                 required: True
                 seq:
                     - type: scalar
-                desc: List of source directories to backup. Globs are expanded.
+                desc: List of source directories to backup (required). Globs are expanded.
                 example:
                     - /home
                     - /etc
@@ -28,16 +28,15 @@ map:
             repository:
                 required: True
                 type: scalar
-                desc: Path to local or remote repository.
+                desc: Path to local or remote repository (required).
                 example: user@backupserver:sourcehostname.borg
             exclude_patterns:
                 seq:
                     - type: scalar
                 desc: |
-                    Exclude patterns. Any paths matching these patterns are excluded from backups.
-                    Globs are expanded. See
-                    https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns for
-                    details.
+                    Any paths matching these patterns are excluded from backups. Globs are expanded.
+                    See https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns
+                    for details.
                 example:
                     - '*.pyc'
                     - /home/*/.cache
diff --git a/sample/config b/sample/config
deleted file mode 100644
index e778aaacd..000000000
--- a/sample/config
+++ /dev/null
@@ -1,48 +0,0 @@
-[location]
-# Space-separated list of source directories to backup.
-# Globs are expanded.
-source_directories: /home /etc /var/log/syslog*
-
-# Stay in same file system (do not cross mount points).
-#one_file_system: True
-
-# Alternate Borg remote executable (defaults to "borg"):
-#remote_path: borg1
-
-# Path to local or remote repository.
-repository: user@backupserver:sourcehostname.borg
-
-[storage]
-# Passphrase to unlock the encryption key with. Only use on repositories that
-# were initialized with passphrase/repokey encryption.
-#encryption_passphrase: foo
-
-# Type of compression to use when creating archives. See
-# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
-# for details. Defaults to no compression.
-#compression: lz4
-
-# Umask to be used for borg create.
-#umask: 0077
-
-[retention]
-# Retention policy for how many backups to keep in each category. See
-# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
-#keep_within: 3H
-#keep_hourly: 24
-keep_daily: 7
-keep_weekly: 4
-keep_monthly: 6
-keep_yearly: 1
-
-#prefix: sourcehostname
-
-[consistency]
-# Space-separated list of consistency checks to run: "repository", "archives",
-# or both. Defaults to both. Set to "disabled" to disable all consistency
-# checks. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
-# for details.
-checks: repository archives
-
-# Restrict the number of checked archives to the last n.
-#check_last: 3
diff --git a/sample/config.yaml b/sample/config.yaml
deleted file mode 100644
index daac2305f..000000000
--- a/sample/config.yaml
+++ /dev/null
@@ -1,56 +0,0 @@
-location:
-    # List of source directories to backup. Globs are expanded.
-    source_directories:
-        - /home
-        - /etc
-        - /var/log/syslog*
-
-    # Stay in same file system (do not cross mount points).
-    #one_file_system: yes
-
-    # Alternate Borg remote executable (defaults to "borg"):
-    #remote_path: borg1
-
-    # Path to local or remote repository.
-    repository: user@backupserver:sourcehostname.borg
-
-#storage:
-    # Passphrase to unlock the encryption key with. Only use on repositories
-    # that were initialized with passphrase/repokey encryption. Quote the value
-    # if it contains punctuation so it parses correctly. And backslash any
-    # quote or backslash literals as well.
-    #encryption_passphrase: "foo"
-
-    # Type of compression to use when creating archives. See
-    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
-    # for details. Defaults to no compression.
-    #compression: lz4
-
-    # Umask to be used for borg create.
-    #umask: 0077
-
-retention:
-    # Retention policy for how many backups to keep in each category. See
-    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for
-    # details.
-    #keep_within: 3H
-    #keep_hourly: 24
-    keep_daily: 7
-    keep_weekly: 4
-    keep_monthly: 6
-    keep_yearly: 1
-
-    # When pruning, only consider archive names starting with this prefix.
-    #prefix: sourcehostname
-
-consistency:
-    # List of consistency checks to run: "repository", "archives", or both.
-    # Defaults to both. Set to "disabled" to disable all consistency checks.
-    # See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
-    # for details.
-    checks:
-        - repository
-        - archives
-
-    # Restrict the number of checked archives to the last n.
-    #check_last: 3
diff --git a/sample/excludes b/sample/excludes
deleted file mode 100644
index 7e81c882a..000000000
--- a/sample/excludes
+++ /dev/null
@@ -1,3 +0,0 @@
-*.pyc
-/home/*/.cache
-/etc/ssl

From b3d0fb0cee97ced1e2f3cbdd54917d06eddad317 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 15:20:50 -0700
Subject: [PATCH 137/189] When writing config, make containing directory if
 necessary. Also default to tighter permissions.

---
 borgmatic/commands/borgmatic.py                    |  2 +-
 borgmatic/commands/convert_config.py               |  7 +++++--
 borgmatic/config/generate.py                       | 11 +++++++++--
 .../tests/integration/config/test_generate.py      | 14 ++++++++++++--
 4 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 1fcc582d0..a1d53e3fc 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -42,7 +42,7 @@ def parse_arguments(*arguments):
 def main():  # pragma: no cover
     try:
         # TODO: Detect whether only legacy config is present. If so, inform the user about how to
-        # upgrade, then exet.
+        # upgrade, then exit.
 
         args = parse_arguments(*sys.argv[1:])
         config = parse_configuration(args.config_filename, schema_filename())
diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py
index eb0bfa876..5e61a7706 100644
--- a/borgmatic/commands/convert_config.py
+++ b/borgmatic/commands/convert_config.py
@@ -87,8 +87,11 @@ def main():  # pragma: no cover
 
         destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema)
 
-        generate.write_configuration(args.destination_config_filename, destination_config)
-        os.chmod(args.destination_config_filename, source_config_file_mode)
+        generate.write_configuration(
+            args.destination_config_filename,
+            destination_config,
+            mode=source_config_file_mode,
+        )
 
         # TODO: As a backstop, check that the written config can actually be read and parsed, and
         # that it matches the destination config data structure that was written.
diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py
index 8b552d5b7..626747fd4 100644
--- a/borgmatic/config/generate.py
+++ b/borgmatic/config/generate.py
@@ -40,17 +40,24 @@ def _schema_to_sample_configuration(schema, level=0):
     return config
 
 
-def write_configuration(config_filename, config):
+def write_configuration(config_filename, config, mode=0o600):
     '''
     Given a target config filename and a config data structure of nested OrderedDicts, write out the
-    config to file as YAML.
+    config to file as YAML. Create any containing directories as needed.
     '''
     if os.path.exists(config_filename):
         raise FileExistsError('{} already exists. Aborting.'.format(config_filename))
 
+    try:
+        os.makedirs(os.path.dirname(config_filename), mode=0o700)
+    except FileExistsError:
+        pass
+
     with open(config_filename, 'w') as config_file:
         config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
 
+    os.chmod(config_filename, mode)
+
 
 def add_comments_to_configuration(config, schema, indent=0):
     '''
diff --git a/borgmatic/tests/integration/config/test_generate.py b/borgmatic/tests/integration/config/test_generate.py
index 078367d35..048315a22 100644
--- a/borgmatic/tests/integration/config/test_generate.py
+++ b/borgmatic/tests/integration/config/test_generate.py
@@ -18,21 +18,31 @@ def test_insert_newline_before_comment_does_not_raise():
 
 def test_write_configuration_does_not_raise():
     flexmock(os.path).should_receive('exists').and_return(False)
+    flexmock(os).should_receive('makedirs')
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').and_return(StringIO())
+    flexmock(os).should_receive('chmod')
 
     module.write_configuration('config.yaml', {})
 
 
 def test_write_configuration_with_already_existing_file_raises():
     flexmock(os.path).should_receive('exists').and_return(True)
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').and_return(StringIO())
 
     with pytest.raises(FileExistsError):
         module.write_configuration('config.yaml', {})
 
 
+def test_write_configuration_with_already_existing_directory_does_not_raise():
+    flexmock(os.path).should_receive('exists').and_return(False)
+    flexmock(os).should_receive('makedirs').and_raise(FileExistsError)
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').and_return(StringIO())
+    flexmock(os).should_receive('chmod')
+
+    module.write_configuration('config.yaml', {})
+
+
 def test_add_comments_to_configuration_does_not_raise():
     # Ensure that it can deal with fields both in the schema and missing from the schema.
     config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])

From 8bf07e476620f47597b5ff6962b9387237e85f82 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 16:06:02 -0700
Subject: [PATCH 138/189] Provide helpful message when borgmatic is run with
 only legacy config present.

---
 README.md                                   |  2 +-
 borgmatic/commands/borgmatic.py             |  9 +++---
 borgmatic/config/convert.py                 | 29 +++++++++++++++++++
 borgmatic/tests/unit/config/test_convert.py | 31 +++++++++++++++++++++
 4 files changed, 65 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index b1225c346..6c25de2e8 100644
--- a/README.md
+++ b/README.md
@@ -97,7 +97,7 @@ upgrade to the new version of borgmatic:
 
 Then, run:
 
-    sudo upgrade-borgmatic-configuration
+    sudo upgrade-borgmatic-config
 
 That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
 (by default) using the values from both your existing configuration and
diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index a1d53e3fc..0299da69d 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -5,9 +5,10 @@ from subprocess import CalledProcessError
 import sys
 
 from borgmatic import borg
-from borgmatic.config.validate import parse_configuration, schema_filename
+from borgmatic.config import convert, validate
 
 
+LEGACY_CONFIG_FILENAME = '/etc/borgmatic/config'
 DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 
@@ -41,11 +42,9 @@ def parse_arguments(*arguments):
 
 def main():  # pragma: no cover
     try:
-        # TODO: Detect whether only legacy config is present. If so, inform the user about how to
-        # upgrade, then exit.
-
         args = parse_arguments(*sys.argv[1:])
-        config = parse_configuration(args.config_filename, schema_filename())
+        convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
+        config = validate.parse_configuration(args.config_filename, validate.schema_filename())
         repository = config.location['repository']
         remote_path = config.location.get('remote_path')
 
diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py
index 11618e958..bdf9eb2d3 100644
--- a/borgmatic/config/convert.py
+++ b/borgmatic/config/convert.py
@@ -1,3 +1,5 @@
+import os
+
 from ruamel import yaml
 
 from borgmatic.config import generate
@@ -48,3 +50,30 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         )
 
     return destination_config
+
+
+class LegacyConfigurationNotUpgraded(FileNotFoundError):
+    def __init__(self):
+        super(LegacyConfigurationNotUpgraded, self).__init__(
+            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
+to YAML. This better supports validation, and has a more natural way to express
+lists of values. To upgrade your existing configuration, run:
+
+    sudo upgrade-borgmatic-config
+
+That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
+(by default) using the values from both your existing configuration and excludes
+files. The new version of borgmatic will consume the YAML configuration file
+instead of the old one.'''
+        )
+
+
+def guard_configuration_upgraded(source_config_filename, destination_config_filename):
+    '''
+    If legacy souce configuration exists but destination upgraded config doesn't, raise
+    LegacyConfigurationNotUpgraded.
+
+    The idea is that we want to alert the user about upgrading their config if they haven't already.
+    '''
+    if os.path.exists(source_config_filename) and not os.path.exists(destination_config_filename):
+        raise LegacyConfigurationNotUpgraded()
diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py
index a88ed2f0c..2e5849bd0 100644
--- a/borgmatic/tests/unit/config/test_convert.py
+++ b/borgmatic/tests/unit/config/test_convert.py
@@ -1,6 +1,8 @@
 from collections import defaultdict, OrderedDict, namedtuple
+import os
 
 from flexmock import flexmock
+import pytest
 
 from borgmatic.config import convert as module
 
@@ -61,3 +63,32 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
         ('retention', OrderedDict()),
         ('consistency', OrderedDict([('checks', ['repository', 'archives'])])),
     ])
+
+
+def test_guard_configuration_upgraded_raises_when_only_source_config_present():
+    flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+
+    with pytest.raises(module.LegacyConfigurationNotUpgraded):
+        module.guard_configuration_upgraded('config', 'config.yaml')
+
+
+def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present():
+    flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True)
+
+    module.guard_configuration_upgraded('config', 'config.yaml')
+
+
+def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present():
+    flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True)
+
+    module.guard_configuration_upgraded('config', 'config.yaml')
+
+
+def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present():
+    flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+
+    module.guard_configuration_upgraded('config', 'config.yaml')

From 0691cda46fe74705e8e3855fd4d926fb74e583ea Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 16:07:07 -0700
Subject: [PATCH 139/189] Mention generate-borgmatic-config in changelog.

---
 NEWS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/NEWS b/NEWS
index 3209b41ff..e3394d424 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,7 @@
 1.1.0.dev0
 
  * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade.
+ * Added generate-borgmatic-config command for initial config creation.
  * Dropped Python 2 support. Now Python 3 only.
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.

From 8ef6c6fcbe222471eb8efed653b296fab367ce57 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 16:25:13 -0700
Subject: [PATCH 140/189] Bail if "--excludes" argument is provided, as it's
 now deprecated in favor of configuration file.

---
 borgmatic/commands/borgmatic.py               |  5 ++-
 borgmatic/config/convert.py                   | 18 +++++++++++
 .../integration/commands/test_borgmatic.py    | 32 ++-----------------
 borgmatic/tests/unit/config/test_convert.py   |  9 ++++++
 4 files changed, 31 insertions(+), 33 deletions(-)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 0299da69d..33b156ae4 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -28,8 +28,7 @@ def parse_arguments(*arguments):
     parser.add_argument(
         '--excludes',
         dest='excludes_filename',
-        default=DEFAULT_EXCLUDES_FILENAME if os.path.exists(DEFAULT_EXCLUDES_FILENAME) else None,
-        help='Excludes filename',
+        help='Excludes filename, deprecated in favor of excludes_patterns within configuration',
     )
     parser.add_argument(
         '-v', '--verbosity',
@@ -46,7 +45,7 @@ def main():  # pragma: no cover
         convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
         config = validate.parse_configuration(args.config_filename, validate.schema_filename())
         repository = config.location['repository']
-        remote_path = config.location.get('remote_path')
+        remote_path = config.location['remote_path']
 
         borg.initialize(config.storage)
         borg.create_archive(
diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py
index bdf9eb2d3..1aaf7aa29 100644
--- a/borgmatic/config/convert.py
+++ b/borgmatic/config/convert.py
@@ -77,3 +77,21 @@ def guard_configuration_upgraded(source_config_filename, destination_config_file
     '''
     if os.path.exists(source_config_filename) and not os.path.exists(destination_config_filename):
         raise LegacyConfigurationNotUpgraded()
+
+
+class LegacyExcludesFilenamePresent(FileNotFoundError):
+    def __init__(self):
+        super(LegacyExcludesFilenamePresent, self).__init__(
+            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
+to YAML. This better supports validation, and has a more natural way to express
+lists of values. The new configuration file incorporates excludes, so you no
+longer need to provide an excludes filename on the command-line with an
+"--excludes" argument.
+
+Please remove the "--excludes" argument and run borgmatic again.'''
+        )
+
+
+def guard_excludes_filename_omitted(excludes_filename):
+    if excludes_filename != None:
+        raise LegacyExcludesFilenamePresent()
diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py
index 12afeedf0..f1cc43a6a 100644
--- a/borgmatic/tests/integration/commands/test_borgmatic.py
+++ b/borgmatic/tests/integration/commands/test_borgmatic.py
@@ -7,18 +7,14 @@ from borgmatic.commands import borgmatic as module
 
 
 def test_parse_arguments_with_no_arguments_uses_defaults():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
     parser = module.parse_arguments()
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+    assert parser.excludes_filename == None
     assert parser.verbosity is None
 
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
     parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
 
     assert parser.config_filename == 'myconfig'
@@ -26,38 +22,14 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
     assert parser.verbosity is None
 
 
-def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
-    flexmock(os.path).should_receive('exists').and_return(False)
-
-    parser = module.parse_arguments()
-
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename is None
-    assert parser.verbosity is None
-
-
-def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
-    flexmock(os.path).should_receive('exists').and_return(False)
-
-    parser = module.parse_arguments('--excludes', 'myexcludes')
-
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity is None
-
-
 def test_parse_arguments_with_verbosity_flag_overrides_default():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
     parser = module.parse_arguments('--verbosity', '1')
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+    assert parser.excludes_filename == None
     assert parser.verbosity == 1
 
 
 def test_parse_arguments_with_invalid_arguments_exits():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
     with pytest.raises(SystemExit):
         module.parse_arguments('--posix-me-harder')
diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py
index 2e5849bd0..39f0cee0e 100644
--- a/borgmatic/tests/unit/config/test_convert.py
+++ b/borgmatic/tests/unit/config/test_convert.py
@@ -92,3 +92,12 @@ def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present
     flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
 
     module.guard_configuration_upgraded('config', 'config.yaml')
+
+
+def test_guard_excludes_filename_omitted_raises_when_filename_provided():
+    with pytest.raises(module.LegacyExcludesFilenamePresent):
+        module.guard_excludes_filename_omitted(excludes_filename='/etc/borgmatic/excludes')
+
+
+def test_guard_excludes_filename_omitted_does_not_raise_when_filename_not_provided():
+    module.guard_excludes_filename_omitted(excludes_filename=None)

From 41d202c2e7e4b4c6c9c987f81f30bdacd0c48b13 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 10 Jul 2017 16:26:32 -0700
Subject: [PATCH 141/189] TODO about using the new exclude_patterns.

---
 borgmatic/commands/borgmatic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 33b156ae4..c2083d969 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -28,7 +28,7 @@ def parse_arguments(*arguments):
     parser.add_argument(
         '--excludes',
         dest='excludes_filename',
-        help='Excludes filename, deprecated in favor of excludes_patterns within configuration',
+        help='Excludes filename, deprecated in favor of exclude_patterns within configuration',
     )
     parser.add_argument(
         '-v', '--verbosity',
@@ -48,6 +48,7 @@ def main():  # pragma: no cover
         remote_path = config.location['remote_path']
 
         borg.initialize(config.storage)
+        # TODO: Use the new exclude_patterns.
         borg.create_archive(
             args.excludes_filename, args.verbosity, config.storage, **config.location
         )

From edb54b300bac377fda0e8a48747edea06cfd46b5 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 20:11:49 -0700
Subject: [PATCH 142/189] Fixing up borg module to deal with new parsed config
 file structures.

---
 borgmatic/borg.py                 |  59 +++++++++++------
 borgmatic/commands/borgmatic.py   |   5 +-
 borgmatic/tests/unit/test_borg.py | 103 ++++++++++++++++--------------
 3 files changed, 95 insertions(+), 72 deletions(-)

diff --git a/borgmatic/borg.py b/borgmatic/borg.py
index f4c5ba947..22d3032f7 100644
--- a/borgmatic/borg.py
+++ b/borgmatic/borg.py
@@ -1,10 +1,11 @@
 from datetime import datetime
+import glob
+import itertools
 import os
-import re
 import platform
+import re
 import subprocess
-from glob import glob
-from itertools import chain
+import tempfile
 
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
@@ -22,18 +23,38 @@ def initialize(storage_config, command=COMMAND):
         os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase
 
 
+def _write_exclude_file(exclude_patterns=None):
+    '''
+    Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
+    None if no patterns are provided.
+    '''
+    if not exclude_patterns:
+        return None
+
+    exclude_file = tempfile.NamedTemporaryFile('w')
+    exclude_file.write('\n'.join(exclude_patterns))
+    exclude_file.flush()
+
+    return exclude_file
+
+
 def create_archive(
-    excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND,
-    one_file_system=None, remote_path=None,
+    verbosity, storage_config, source_directories, repository, exclude_patterns=None,
+    command=COMMAND, one_file_system=None, remote_path=None,
 ):
     '''
-    Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
-    list of source directories, a local or remote repository path, and a command to run, create an
-    attic archive.
+    Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
+    repository path, a list of exclude patterns, and a command to run, create an attic archive.
     '''
-    sources = re.split('\s+', source_directories)
-    sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources))
-    exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
+    sources = tuple(
+        itertools.chain.from_iterable(
+            glob.glob(directory) or [directory]
+            for directory in source_directories
+        )
+    )
+
+    exclude_file = _write_exclude_file(exclude_patterns)
+    exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
     umask = storage_config.get('umask', None)
@@ -109,12 +130,11 @@ DEFAULT_CHECKS = ('repository', 'archives')
 
 def _parse_checks(consistency_config):
     '''
-    Given a consistency config with a space-separated "checks" option, transform it to a tuple of
-    named checks to run.
+    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
 
     For example, given a retention config of:
 
-        {'checks': 'repository archives'}
+        {'checks': ['repository', 'archives']}
 
     This will be returned as:
 
@@ -123,14 +143,11 @@ def _parse_checks(consistency_config):
     If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
     "disabled", return an empty tuple, meaning that no checks should be run.
     '''
-    checks = consistency_config.get('checks', '').strip()
-    if not checks:
-        return DEFAULT_CHECKS
+    checks = consistency_config.get('checks', [])
+    if checks == ['disabled']:
+        return ()
 
-    return tuple(
-        check for check in consistency_config['checks'].split(' ')
-        if check.lower() not in ('disabled', '')
-    )
+    return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
 
 
 def _make_check_flags(checks, check_last=None):
diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index c2083d969..9bde8bc4b 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -48,10 +48,7 @@ def main():  # pragma: no cover
         remote_path = config.location['remote_path']
 
         borg.initialize(config.storage)
-        # TODO: Use the new exclude_patterns.
-        borg.create_archive(
-            args.excludes_filename, args.verbosity, config.storage, **config.location
-        )
+        borg.create_archive(args.verbosity, config.storage, **config.location)
         borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path)
         borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:
diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py
index 3c2aa1c71..57e75a312 100644
--- a/borgmatic/tests/unit/test_borg.py
+++ b/borgmatic/tests/unit/test_borg.py
@@ -30,6 +30,20 @@ def test_initialize_without_passphrase_should_not_set_environment():
     finally:
         os.environ = orig_environ
 
+def test_write_exclude_file_does_not_raise():
+    temporary_file = flexmock(
+        name='filename',
+        write=lambda mode: None,
+        flush=lambda: None,
+    )
+    flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
+
+    module._write_exclude_file(['exclude'])
+
+
+def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
+    module._write_exclude_file([])
+
 
 def insert_subprocess_mock(check_call_command, **kwargs):
     subprocess = flexmock(STDOUT=STDOUT)
@@ -53,110 +67,100 @@ def insert_datetime_mock():
     ).mock
 
 
-CREATE_COMMAND_WITHOUT_EXCLUDES = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
-CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes')
+CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
 
 
 def test_create_archive_should_call_borg_with_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND)
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
-def test_create_archive_with_two_spaces_in_source_directories():
-    insert_subprocess_mock(CREATE_COMMAND)
+def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
+    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes'))
+    insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=['exclude'],
         verbosity=None,
         storage_config={},
-        source_directories='foo  bar',
-        repository='repo',
-        command='borg',
-    )
-
-
-def test_create_archive_with_none_excludes_filename_should_call_borg_without_excludes():
-    insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename=None,
-        verbosity=None,
-        storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=VERBOSITY_SOME,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=VERBOSITY_LOTS,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={'compression': 'rle'},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
         one_file_system=True,
@@ -164,15 +168,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
 
 
 def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
         remote_path='borg1',
@@ -180,63 +185,67 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
 
 
 def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={'umask': 740},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_source_directories_glob_expands():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
-    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
-        excludes_filename=None,
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo*',
+        source_directories=['foo*'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*'))
     insert_platform_mock()
     insert_datetime_mock()
-    flexmock(module).should_receive('glob').with_args('foo*').and_return([])
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
     module.create_archive(
-        excludes_filename=None,
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo*',
+        source_directories=['foo*'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
-    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
-        excludes_filename=None,
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo*',
+        source_directories=['foo*'],
         repository='repo',
         command='borg',
     )
@@ -329,7 +338,7 @@ def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parame
 
 
 def test_parse_checks_returns_them_as_tuple():
-    checks = module._parse_checks({'checks': 'foo disabled bar'})
+    checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']})
 
     assert checks == ('foo', 'bar')
 
@@ -341,13 +350,13 @@ def test_parse_checks_with_missing_value_returns_defaults():
 
 
 def test_parse_checks_with_blank_value_returns_defaults():
-    checks = module._parse_checks({'checks': ''})
+    checks = module._parse_checks({'checks': []})
 
     assert checks == module.DEFAULT_CHECKS
 
 
 def test_parse_checks_with_disabled_returns_no_checks():
-    checks = module._parse_checks({'checks': 'disabled'})
+    checks = module._parse_checks({'checks': ['disabled']})
 
     assert checks == ()
 

From 8bfffd8cf7c012a4e7ae899c9a8b79e2cba314f2 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 20:31:26 -0700
Subject: [PATCH 143/189] Removing TODO that basically entails testing
 ruamel.yaml round-tripping, which in theory already has its own tests.

---
 borgmatic/commands/convert_config.py | 3 ---
 borgmatic/config/generate.py         | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py
index 5e61a7706..ca001e0ca 100644
--- a/borgmatic/commands/convert_config.py
+++ b/borgmatic/commands/convert_config.py
@@ -93,9 +93,6 @@ def main():  # pragma: no cover
             mode=source_config_file_mode,
         )
 
-        # TODO: As a backstop, check that the written config can actually be read and parsed, and
-        # that it matches the destination config data structure that was written.
-
         display_result(args)
     except (ValueError, OSError) as error:
         print(error, file=sys.stderr)
diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py
index 626747fd4..1a6d1a86e 100644
--- a/borgmatic/config/generate.py
+++ b/borgmatic/config/generate.py
@@ -50,7 +50,7 @@ def write_configuration(config_filename, config, mode=0o600):
 
     try:
         os.makedirs(os.path.dirname(config_filename), mode=0o700)
-    except FileExistsError:
+    except (FileExistsError, FileNotFoundError):
         pass
 
     with open(config_filename, 'w') as config_file:

From 919d7573c3059e1b519fbd4b1d27f497ed414167 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 20:52:29 -0700
Subject: [PATCH 144/189] Upgrading instructions to super clarify Python 3
 upgrade.

---
 README.md | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 6c25de2e8..c87c2f746 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ To install borgmatic, run the following command to download and install it:
     sudo pip install --upgrade borgmatic
 
 Make sure you're using Python 3, as borgmatic does not support Python 2. (You
-may have to use "pip3" instead of "pip".)
+may have to use "pip3" or similar instead of "pip".)
 
 Then, generate a sample configuration file:
 
@@ -84,8 +84,12 @@ following:
 
     sudo pip install --upgrade borgmatic
 
+(You may have to use "pip3" or similar instead of "pip", so Python 3 gets
+used.)
+
 However, see below about special cases.
 
+
 ### Upgrading from borgmatic 1.0.x
 
 borgmatic changed its configuration file format in version 1.1.0 from
@@ -93,9 +97,22 @@ INI-style to YAML. This better supports validation, and has a more natural way
 to express lists of values. To upgrade your existing configuration, first
 upgrade to the new version of borgmatic:
 
+As of version 1.1.0, borgmatic no longer supports Python 2. If you were
+already running borgmatic with Python 3, then you can simply upgrade borgmatic
+in-place:
+
     sudo pip install --upgrade borgmatic
 
-Then, run:
+But if you were running borgmatic with Python 2, uninstall and reinstall instead:
+
+    sudo pip uninstall borgmatic
+    sudo pip3 install borgmatic
+
+The pip binary names for different versions of Python can differ, so the above
+commands may need some tweaking to work on your machine.
+
+
+Once borgmatic is upgraded, run:
 
     sudo upgrade-borgmatic-config
 

From 3cccac8cb17a908bf137d3db7e67e70f2be57759 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 21:07:09 -0700
Subject: [PATCH 145/189] Mentioning libyaml compile errors in troubleshooting.

---
 README.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/README.md b/README.md
index c87c2f746..feebbdc69 100644
--- a/README.md
+++ b/README.md
@@ -232,6 +232,17 @@ This should make the client keep the connection alive while validating
 backups.
 
 
+### libyaml compilation errors
+
+borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
+use a C YAML library (libyaml) if present. But if it's not installed, then
+when installing or upgrading borgmatic, you may see errors about compiling the
+YAML library. If so, not to worry. borgmatic should install and function
+correctly even without the C YAML library. And borgmatic won't be any faster
+with the C library present, so you don't need to go out of your way to install
+it.
+
+
 ## Issues and feedback
 
 Got an issue or an idea for a feature enhancement? Check out the [borgmatic

From 6af53d11636826abcf105188d2d8b08ae49e8100 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 21:19:26 -0700
Subject: [PATCH 146/189] Fixing gets on config group names.

---
 borgmatic/commands/borgmatic.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 9bde8bc4b..c37b8915d 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -44,13 +44,17 @@ def main():  # pragma: no cover
         args = parse_arguments(*sys.argv[1:])
         convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
         config = validate.parse_configuration(args.config_filename, validate.schema_filename())
-        repository = config.location['repository']
-        remote_path = config.location['remote_path']
+        repository = config['location']['repository']
+        remote_path = config['location']['remote_path']
+        (storage, retention, consistency) = (
+            config.get(group_name, {})
+            for group_name in ('storage', 'retention', 'consistency')
+        )
 
-        borg.initialize(config.storage)
-        borg.create_archive(args.verbosity, config.storage, **config.location)
-        borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path)
-        borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path)
+        borg.initialize(storage)
+        borg.create_archive(args.verbosity, storage, **config['location'])
+        borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+        borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

From f5abe05ce96c1645bb9f0a68dbfbe3da723bb4c0 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 21:20:48 -0700
Subject: [PATCH 147/189] Instructions to make cron file executable.

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index feebbdc69..0d2593692 100644
--- a/README.md
+++ b/README.md
@@ -180,6 +180,7 @@ file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/cron/borgmatic).
 Then, from the directory where you downloaded it:
 
     sudo mv borgmatic /etc/cron.d/borgmatic
+    sudo chmod +x /etc/cron.d/borgmatic
 
 You can modify the cron file if you'd like to run borgmatic more or less frequently.
 

From e5c12fc81c982d6e77ba7c0858284326b07ca79e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 21:23:01 -0700
Subject: [PATCH 148/189] Mentioning test coverage addition in NEWS.

---
 NEWS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/NEWS b/NEWS
index e3394d424..0941a1c62 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,7 @@
  * Dropped Python 2 support. Now Python 3 only.
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.
+ * Enabled test coverage output during tox runs.
  * Added logo.
 
 1.0.3

From cd8ceccfafbdba6536cefce660115954ee122dd3 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 21:50:29 -0700
Subject: [PATCH 149/189] To free up space, now pruning backups prior to
 creating a new backup.

---
 NEWS                            | 1 +
 borgmatic/commands/borgmatic.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 0941a1c62..47e53617c 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,7 @@
  * Dropped Python 2 support. Now Python 3 only.
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.
+ * To free up space, now pruning backups prior to creating a new backup.
  * Enabled test coverage output during tox runs.
  * Added logo.
 
diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index c37b8915d..90fcccae1 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -52,8 +52,8 @@ def main():  # pragma: no cover
         )
 
         borg.initialize(storage)
-        borg.create_archive(args.verbosity, storage, **config['location'])
         borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+        borg.create_archive(args.verbosity, storage, **config['location'])
         borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)

From 90a0d3b1e02d6d7dc7d33065a69360215817d769 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 22:17:37 -0700
Subject: [PATCH 150/189] Renaming group to section for consistency.

---
 borgmatic/commands/borgmatic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 90fcccae1..f1f9c31d4 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -47,8 +47,8 @@ def main():  # pragma: no cover
         repository = config['location']['repository']
         remote_path = config['location']['remote_path']
         (storage, retention, consistency) = (
-            config.get(group_name, {})
-            for group_name in ('storage', 'retention', 'consistency')
+            config.get(section_name, {})
+            for section_name in ('storage', 'retention', 'consistency')
         )
 
         borg.initialize(storage)

From ee3edeaac2239b9cefb67ced6ea3f6a6f0c7c27e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 22:56:46 -0700
Subject: [PATCH 151/189] Support for backing up to multiple repositories.

---
 NEWS                                          |   1 +
 README.md                                     |   5 +-
 borgmatic/borg.py                             |  10 +-
 borgmatic/commands/borgmatic.py               |  20 ++--
 borgmatic/config/convert.py                   |   9 +-
 borgmatic/config/schema.yaml                  |  12 +-
 .../tests/integration/config/test_validate.py |  13 ++-
 borgmatic/tests/unit/config/test_convert.py   |   5 +-
 borgmatic/tests/unit/test_borg.py             | 103 ++++++++++++------
 9 files changed, 115 insertions(+), 63 deletions(-)

diff --git a/NEWS b/NEWS
index 47e53617c..447317047 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,7 @@
  * Dropped Python 2 support. Now Python 3 only.
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.
+ * Support for backing up to multiple repositories.
  * To free up space, now pruning backups prior to creating a new backup.
  * Enabled test coverage output during tox runs.
  * Added logo.
diff --git a/README.md b/README.md
index 0d2593692..0bd5cda64 100644
--- a/README.md
+++ b/README.md
@@ -21,8 +21,9 @@ location:
         - /etc
         - /var/log/syslog*
 
-    # Path to local or remote repository.
-    repository: user@backupserver:sourcehostname.borg
+    # Paths to local or remote repositories.
+    repositories:
+        - user@backupserver:sourcehostname.borg
 
     # Any paths matching these patterns are excluded from backups.
     exclude_patterns:
diff --git a/borgmatic/borg.py b/borgmatic/borg.py
index 22d3032f7..f42353d21 100644
--- a/borgmatic/borg.py
+++ b/borgmatic/borg.py
@@ -39,8 +39,7 @@ def _write_exclude_file(exclude_patterns=None):
 
 
 def create_archive(
-    verbosity, storage_config, source_directories, repository, exclude_patterns=None,
-    command=COMMAND, one_file_system=None, remote_path=None,
+    verbosity, repository, location_config, storage_config, command=COMMAND,
 ):
     '''
     Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
@@ -49,17 +48,18 @@ def create_archive(
     sources = tuple(
         itertools.chain.from_iterable(
             glob.glob(directory) or [directory]
-            for directory in source_directories
+            for directory in location_config['source_directories']
         )
     )
 
-    exclude_file = _write_exclude_file(exclude_patterns)
+    exclude_file = _write_exclude_file(location_config.get('exclude_patterns'))
     exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
     umask = storage_config.get('umask', None)
     umask_flags = ('--umask', str(umask)) if umask else ()
-    one_file_system_flags = ('--one-file-system',) if one_file_system else ()
+    one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
+    remote_path = location_config.get('remote_path')
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--info', '--stats',),
diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index f1f9c31d4..a5f39a024 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -44,17 +44,23 @@ def main():  # pragma: no cover
         args = parse_arguments(*sys.argv[1:])
         convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
         config = validate.parse_configuration(args.config_filename, validate.schema_filename())
-        repository = config['location']['repository']
-        remote_path = config['location']['remote_path']
-        (storage, retention, consistency) = (
+        (location, storage, retention, consistency) = (
             config.get(section_name, {})
-            for section_name in ('storage', 'retention', 'consistency')
+            for section_name in ('location', 'storage', 'retention', 'consistency')
         )
+        remote_path = location.get('remote_path')
 
         borg.initialize(storage)
-        borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
-        borg.create_archive(args.verbosity, storage, **config['location'])
-        borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
+
+        for repository in location['repositories']:
+            borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+            borg.create_archive(
+                args.verbosity,
+                repository,
+                location,
+                storage,
+            )
+            borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py
index 1aaf7aa29..832d7d310 100644
--- a/borgmatic/config/convert.py
+++ b/borgmatic/config/convert.py
@@ -32,9 +32,12 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         for section_name, section_config in source_config._asdict().items()
     ])
 
-    # Split space-seperated values into actual lists, and merge in excludes.
-    destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
-    destination_config['location']['exclude_patterns'] = source_excludes
+    # Split space-seperated values into actual lists, make "repository" into a list, and merge in
+    # excludes.
+    location = destination_config['location']
+    location['source_directories'] = source_config.location['source_directories'].split(' ')
+    location['repositories'] = [location.pop('repository')]
+    location['exclude_patterns'] = source_excludes
 
     if source_config.consistency['checks']:
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index dfd0c55bb..64fed0794 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -25,11 +25,15 @@ map:
                 type: scalar
                 desc: Alternate Borg remote executable. Defaults to "borg".
                 example: borg1
-            repository:
+            repositories:
                 required: True
-                type: scalar
-                desc: Path to local or remote repository (required).
-                example: user@backupserver:sourcehostname.borg
+                seq:
+                    - type: scalar
+                desc: |
+                    Paths to local or remote repositories (required). Multiple repositories are
+                    backed up to in sequence.
+                example:
+                    - user@backupserver:sourcehostname.borg
             exclude_patterns:
                 seq:
                     - type: scalar
diff --git a/borgmatic/tests/integration/config/test_validate.py b/borgmatic/tests/integration/config/test_validate.py
index 90e223d13..9b63ccc89 100644
--- a/borgmatic/tests/integration/config/test_validate.py
+++ b/borgmatic/tests/integration/config/test_validate.py
@@ -35,7 +35,8 @@ def test_parse_configuration_transforms_file_into_mapping():
                 - /home
                 - /etc
 
-            repository: hostname.borg
+            repositories:
+                - hostname.borg
 
         retention:
             keep_daily: 7
@@ -50,7 +51,7 @@ def test_parse_configuration_transforms_file_into_mapping():
     result = module.parse_configuration('config.yaml', 'schema.yaml')
 
     assert result == {
-        'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
+        'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']},
         'retention': {'keep_daily': 7},
         'consistency': {'checks': ['repository', 'archives']},
     }
@@ -65,7 +66,8 @@ def test_parse_configuration_passes_through_quoted_punctuation():
             source_directories:
                 - /home
 
-            repository: "{}.borg"
+            repositories:
+                - "{}.borg"
         '''.format(escaped_punctuation)
     )
 
@@ -74,7 +76,7 @@ def test_parse_configuration_passes_through_quoted_punctuation():
     assert result == {
         'location': {
             'source_directories': ['/home'],
-            'repository': '{}.borg'.format(string.punctuation),
+            'repositories': ['{}.borg'.format(string.punctuation)],
         },
     }
 
@@ -105,7 +107,8 @@ def test_parse_configuration_raises_for_validation_error():
         '''
         location:
             source_directories: yes
-            repository: hostname.borg
+            repositories:
+                - hostname.borg
         '''
     )
 
diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py
index 39f0cee0e..827ecdbc2 100644
--- a/borgmatic/tests/unit/config/test_convert.py
+++ b/borgmatic/tests/unit/config/test_convert.py
@@ -28,7 +28,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
             'location',
             OrderedDict([
                 ('source_directories', ['/home']),
-                ('repository', 'hostname.borg'),
+                ('repositories', ['hostname.borg']),
                 ('exclude_patterns', ['/var']),
             ]),
         ),
@@ -41,7 +41,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
 def test_convert_legacy_parsed_config_splits_space_separated_values():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
     source_config = Parsed_config(
-        location=OrderedDict([('source_directories', '/home /etc')]),
+        location=OrderedDict([('source_directories', '/home /etc'), ('repository', 'hostname.borg')]),
         storage=OrderedDict(),
         retention=OrderedDict(),
         consistency=OrderedDict([('checks', 'repository archives')]),
@@ -56,6 +56,7 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
             'location',
             OrderedDict([
                 ('source_directories', ['/home', '/etc']),
+                ('repositories', ['hostname.borg']),
                 ('exclude_patterns', ['/var']),
             ]),
         ),
diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py
index 57e75a312..36cc3121f 100644
--- a/borgmatic/tests/unit/test_borg.py
+++ b/borgmatic/tests/unit/test_borg.py
@@ -77,11 +77,14 @@ def test_create_archive_should_call_borg_with_parameters():
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -93,11 +96,14 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=['exclude'],
         verbosity=None,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': ['exclude'],
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -109,11 +115,14 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=VERBOSITY_SOME,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -125,11 +134,14 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=VERBOSITY_LOTS,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -141,11 +153,14 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={'compression': 'rle'},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'compression': 'rle'},
         command='borg',
     )
 
@@ -157,13 +172,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'one_file_system': True,
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
-        one_file_system=True,
     )
 
 
@@ -174,13 +192,16 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'remote_path': 'borg1',
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
-        remote_path='borg1',
     )
 
 
@@ -191,11 +212,14 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={'umask': 740},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'umask': 740},
         command='borg',
     )
 
@@ -208,11 +232,14 @@ def test_create_archive_with_source_directories_glob_expands():
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo*'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -225,11 +252,14 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo*'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -242,11 +272,14 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo*'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 

From 588955a467c0bbb61692da13f0cd37f7efc93ecc Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 23:27:21 -0700
Subject: [PATCH 152/189] Setting release version.

---
 NEWS     | 2 +-
 setup.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 447317047..8dc4016a7 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
-1.1.0.dev0
+1.1.0
 
  * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade.
  * Added generate-borgmatic-config command for initial config creation.
diff --git a/setup.py b/setup.py
index 8f8da5c24..a8a867843 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.0.dev0'
+VERSION = '1.1.0'
 
 
 setup(

From b61b09f55cc07f96ab8f92a2ed8d4c3287552607 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 22 Jul 2017 23:27:26 -0700
Subject: [PATCH 153/189] Added tag 1.1.0 for changeset 5a003056a8ff

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 06c448a4b..1c2351c95 100644
--- a/.hgtags
+++ b/.hgtags
@@ -30,3 +30,4 @@ dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8
 de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
 9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2
 32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3
+5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0

From f44a7884e60178e050199f314ad107a9874b58a4 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 23 Jul 2017 17:34:17 -0700
Subject: [PATCH 154/189] No longer producing univeral (Python 2 + 3) wheel.

---
 NEWS      | 4 ++++
 setup.cfg | 3 ---
 setup.py  | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/NEWS b/NEWS
index 8dc4016a7..eb863866e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+1.1.1.dev0
+
+ * 
+
 1.1.0
 
  * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade.
diff --git a/setup.cfg b/setup.cfg
index 4dd514250..12871ff0f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,2 @@
 [metadata]
 description-file=README.md
-
-[bdist_wheel]
-universal=1
diff --git a/setup.py b/setup.py
index a8a867843..b89ce2678 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.0'
+VERSION = '1.1.1.dev0'
 
 
 setup(

From b36b923c5d6b6860000e5b6bd42392b66307dc84 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 24 Jul 2017 08:41:02 -0700
Subject: [PATCH 155/189] #32: Fix for upgrade-borgmatic-config converting
 check_last option as a string instead of an integer.

---
 NEWS                                        |  6 ++++--
 borgmatic/config/convert.py                 | 14 ++++++++++----
 borgmatic/tests/unit/config/test_convert.py | 10 ++++++++++
 setup.py                                    |  2 +-
 4 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/NEWS b/NEWS
index eb863866e..a580bdc02 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,8 @@
-1.1.1.dev0
+1.1.1
 
- * 
+ * #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of an
+   integer.
+ * Fix for upgrade-borgmatic-config erroring when consistency checks option is not present.
 
 1.1.0
 
diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py
index 832d7d310..923ec4846 100644
--- a/borgmatic/config/convert.py
+++ b/borgmatic/config/convert.py
@@ -10,10 +10,16 @@ def _convert_section(source_section_config, section_schema):
     Given a legacy Parsed_config instance for a single section, convert it to its corresponding
     yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.
 
-    Additionally, use the section schema as a source of helpful comments to include within the
-    returned CommentedMap.
+    Where integer types exist in the given section schema, convert their values to integers.
     '''
-    destination_section_config = yaml.comments.CommentedMap(source_section_config)
+    destination_section_config = yaml.comments.CommentedMap([
+        (
+            option_name,
+            int(option_value)
+                if section_schema['map'].get(option_name, {}).get('type') == 'int' else option_value
+        )
+        for option_name, option_value in source_section_config.items()
+    ])
 
     return destination_section_config
 
@@ -39,7 +45,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
     location['repositories'] = [location.pop('repository')]
     location['exclude_patterns'] = source_excludes
 
-    if source_config.consistency['checks']:
+    if source_config.consistency.get('checks'):
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
 
     # Add comments to each section, and then add comments to the fields in each section.
diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py
index 827ecdbc2..58693ab2c 100644
--- a/borgmatic/tests/unit/config/test_convert.py
+++ b/borgmatic/tests/unit/config/test_convert.py
@@ -10,6 +10,16 @@ from borgmatic.config import convert as module
 Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))
 
 
+def test_convert_section_generates_integer_value_for_integer_type_in_schema():
+    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
+    source_section_config = OrderedDict([('check_last', '3')])
+    section_schema = {'map': {'check_last': {'type': 'int'}}}
+
+    destination_config = module._convert_section(source_section_config, section_schema)
+
+    assert destination_config == OrderedDict([('check_last', 3)])
+
+
 def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
     source_config = Parsed_config(
diff --git a/setup.py b/setup.py
index b89ce2678..9796fc4ea 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.1.dev0'
+VERSION = '1.1.1'
 
 
 setup(

From bcd8b9982d307e9ce3671b37e7320064239cec53 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 24 Jul 2017 08:41:05 -0700
Subject: [PATCH 156/189] Added tag 1.1.1 for changeset 7d3d11eff6c0

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 1c2351c95..9273af07d 100644
--- a/.hgtags
+++ b/.hgtags
@@ -31,3 +31,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
 9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2
 32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3
 5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0
+7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1

From 2c61c0bc088d3f9c630a23d71751f463694b7b02 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 24 Jul 2017 19:29:26 -0700
Subject: [PATCH 157/189] #32: Fix for passing check_last as integer to
 subprocess when calling Borg.

---
 NEWS                              | 8 ++++++--
 borgmatic/borg.py                 | 5 ++---
 borgmatic/tests/unit/test_borg.py | 4 ++--
 setup.py                          | 2 +-
 4 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/NEWS b/NEWS
index a580bdc02..a4b6c8a67 100644
--- a/NEWS
+++ b/NEWS
@@ -1,7 +1,11 @@
+1.1.2
+
+ * #32: Fix for passing check_last as integer to subprocess when calling Borg.
+
 1.1.1
 
- * #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of an
-   integer.
+ * Part of #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of
+   an integer.
  * Fix for upgrade-borgmatic-config erroring when consistency checks option is not present.
 
 1.1.0
diff --git a/borgmatic/borg.py b/borgmatic/borg.py
index f42353d21..8c4c73086 100644
--- a/borgmatic/borg.py
+++ b/borgmatic/borg.py
@@ -162,10 +162,9 @@ def _make_check_flags(checks, check_last=None):
 
         ('--repository-only',)
 
-    Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
-    Borg supports this flag.
+    Additionally, if a check_last value is given, a "--last" flag will be added.
     '''
-    last_flag = ('--last', check_last) if check_last else ()
+    last_flag = ('--last', str(check_last)) if check_last else ()
     if checks == DEFAULT_CHECKS:
         return last_flag
 
diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py
index 36cc3121f..4963ab539 100644
--- a/borgmatic/tests/unit/test_borg.py
+++ b/borgmatic/tests/unit/test_borg.py
@@ -409,13 +409,13 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
 def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
     flags = module._make_check_flags(('foo', 'bar'), check_last=3)
 
-    assert flags == ('--foo-only', '--bar-only', '--last', 3)
+    assert flags == ('--foo-only', '--bar-only', '--last', '3')
 
 
 def test_make_check_flags_with_last_returns_last_flag():
     flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
 
-    assert flags == ('--last', 3)
+    assert flags == ('--last', '3')
 
 
 def test_check_archives_should_call_borg_with_parameters():
diff --git a/setup.py b/setup.py
index 9796fc4ea..6d4cc7c36 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.1'
+VERSION = '1.1.2'
 
 
 setup(

From 89cd879529263656aa0ee6119b4caa69f22de923 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Mon, 24 Jul 2017 19:29:28 -0700
Subject: [PATCH 158/189] Added tag 1.1.2 for changeset f052a77a8ad5

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 9273af07d..24e679862 100644
--- a/.hgtags
+++ b/.hgtags
@@ -32,3 +32,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
 32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3
 5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0
 7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1
+f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2

From e3e4aeff9483f709e4513cf759ab865d60d86f93 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 25 Jul 2017 20:32:32 -0700
Subject: [PATCH 159/189] Fix for generate-borgmatic-config writing config with
 invalid one_file_system value.

---
 borgmatic/config/schema.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index 64fed0794..07e245d6f 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -6,10 +6,10 @@ map:
             Where to look for files to backup, and where to store those backups. See
             https://borgbackup.readthedocs.io/en/stable/quickstart.html and
             https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details.
-        required: True
+        required: true
         map:
             source_directories:
-                required: True
+                required: true
                 seq:
                     - type: scalar
                 desc: List of source directories to backup (required). Globs are expanded.
@@ -20,13 +20,13 @@ map:
             one_file_system:
                 type: bool
                 desc: Stay in same file system (do not cross mount points).
-                example: yes
+                example: true
             remote_path:
                 type: scalar
                 desc: Alternate Borg remote executable. Defaults to "borg".
                 example: borg1
             repositories:
-                required: True
+                required: true
                 seq:
                     - type: scalar
                 desc: |
@@ -112,7 +112,7 @@ map:
                 seq:
                     - type: str
                       enum: ['repository', 'archives', 'disabled']
-                      unique: True
+                      unique: true
                 desc: |
                     List of consistency checks to run: "repository", "archives", or both. Defaults
                     to both. Set to "disabled" to disable all consistency checks. See

From 0c8816e6cce58725118743a00971c3a80e1ba437 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 25 Jul 2017 21:18:51 -0700
Subject: [PATCH 160/189] #14: Support for running multiple config files in
 /etc/borgmatic.d/ from a single borgmatic run.

---
 NEWS                                          |  5 ++
 README.md                                     | 23 +++++++-
 borgmatic/commands/borgmatic.py               | 53 +++++++++--------
 borgmatic/config/collect.py                   | 27 +++++++++
 borgmatic/config/convert.py                   | 11 +++-
 .../integration/commands/test_borgmatic.py    | 15 +++--
 borgmatic/tests/unit/config/test_collect.py   | 58 +++++++++++++++++++
 borgmatic/tests/unit/config/test_convert.py   | 16 +++--
 setup.py                                      |  2 +-
 9 files changed, 172 insertions(+), 38 deletions(-)
 create mode 100644 borgmatic/config/collect.py
 create mode 100644 borgmatic/tests/unit/config/test_collect.py

diff --git a/NEWS b/NEWS
index a4b6c8a67..3bc88855e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+1.1.3.dev0
+
+ * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
+ * Fix for generate-borgmatic-config writing config with invalid one_file_system value.
+
 1.1.2
 
  * #32: Fix for passing check_last as integer to subprocess when calling Borg.
diff --git a/README.md b/README.md
index 0bd5cda64..8ea9a8929 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,9 @@ To install borgmatic, run the following command to download and install it:
 Make sure you're using Python 3, as borgmatic does not support Python 2. (You
 may have to use "pip3" or similar instead of "pip".)
 
-Then, generate a sample configuration file:
+## Configuration
+
+After you install borgmatic, generate a sample configuration file:
 
     sudo generate-borgmatic-config
 
@@ -78,6 +80,25 @@ representative. All fields are optional except where indicated, so feel free
 to remove anything you don't need.
 
 
+### Multiple configuration files
+
+A more advanced usage is to create multiple separate configuration files and
+place each one in a /etc/borgmatic.d directory. For instance:
+
+    sudo mkdir /etc/borgmatic.d
+    sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
+    sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
+
+With this approach, you can have entirely different backup policies for
+different applications on your system. For instance, you may want one backup
+configuration for your database data directory, and a different configuration
+for your user home directories.
+
+When you set up multiple configuration files like this, borgmatic will run
+each one in turn from a single borgmatic invocation. This includes, by
+default, the traditional /etc/borgmatic/config.yaml as well.
+
+
 ## Upgrading
 
 In general, all you should need to do to upgrade borgmatic is run the
diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index a5f39a024..a26c30cd6 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -5,12 +5,12 @@ from subprocess import CalledProcessError
 import sys
 
 from borgmatic import borg
-from borgmatic.config import convert, validate
+from borgmatic.config import collect, convert, validate
 
 
-LEGACY_CONFIG_FILENAME = '/etc/borgmatic/config'
-DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
-DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
+LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
+DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d']
+DEFAULT_EXCLUDES_PATH = '/etc/borgmatic/excludes'
 
 
 def parse_arguments(*arguments):
@@ -21,9 +21,10 @@ def parse_arguments(*arguments):
     parser = ArgumentParser()
     parser.add_argument(
         '-c', '--config',
-        dest='config_filename',
-        default=DEFAULT_CONFIG_FILENAME,
-        help='Configuration filename',
+        nargs='+',
+        dest='config_paths',
+        default=DEFAULT_CONFIG_PATHS,
+        help='Configuration filenames or directories, defaults to: {}'.format(' '.join(DEFAULT_CONFIG_PATHS)),
     )
     parser.add_argument(
         '--excludes',
@@ -42,25 +43,31 @@ def parse_arguments(*arguments):
 def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
-        convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
-        config = validate.parse_configuration(args.config_filename, validate.schema_filename())
-        (location, storage, retention, consistency) = (
-            config.get(section_name, {})
-            for section_name in ('location', 'storage', 'retention', 'consistency')
-        )
-        remote_path = location.get('remote_path')
+        config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+        convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
 
-        borg.initialize(storage)
+        if len(config_filenames) == 0:
+            raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths)))
 
-        for repository in location['repositories']:
-            borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
-            borg.create_archive(
-                args.verbosity,
-                repository,
-                location,
-                storage,
+        for config_filename in config_filenames:
+            config = validate.parse_configuration(config_filename, validate.schema_filename())
+            (location, storage, retention, consistency) = (
+                config.get(section_name, {})
+                for section_name in ('location', 'storage', 'retention', 'consistency')
             )
-            borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
+            remote_path = location.get('remote_path')
+
+            borg.initialize(storage)
+
+            for repository in location['repositories']:
+                borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+                borg.create_archive(
+                    args.verbosity,
+                    repository,
+                    location,
+                    storage,
+                )
+                borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py
new file mode 100644
index 000000000..dbe6f2fe7
--- /dev/null
+++ b/borgmatic/config/collect.py
@@ -0,0 +1,27 @@
+import os
+
+
+def collect_config_filenames(config_paths):
+    '''
+    Given a sequence of config paths, both filenames and directories, resolve that to just an
+    iterable of files. Accomplish this by listing any given directories looking for contained config
+    files. This is non-recursive, so any directories within the given directories are ignored.
+
+    Return paths even if they don't exist on disk, so the user can find out about missing
+    configuration paths. However, skip /etc/borgmatic.d if it's missing, so the user doesn't have to
+    create it unless they need it.
+    '''
+    for path in config_paths:
+        exists = os.path.exists(path)
+
+        if os.path.realpath(path) == '/etc/borgmatic.d' and not exists:
+            continue
+
+        if not os.path.isdir(path) or not exists:
+            yield path
+            continue
+
+        for filename in os.listdir(path):
+            full_filename = os.path.join(path, filename)
+            if not os.path.isdir(full_filename):
+                yield full_filename
diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py
index 923ec4846..97e75715b 100644
--- a/borgmatic/config/convert.py
+++ b/borgmatic/config/convert.py
@@ -77,14 +77,19 @@ instead of the old one.'''
         )
 
 
-def guard_configuration_upgraded(source_config_filename, destination_config_filename):
+def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
     '''
-    If legacy souce configuration exists but destination upgraded config doesn't, raise
+    If legacy source configuration exists but no destination upgraded configs do, raise
     LegacyConfigurationNotUpgraded.
 
     The idea is that we want to alert the user about upgrading their config if they haven't already.
     '''
-    if os.path.exists(source_config_filename) and not os.path.exists(destination_config_filename):
+    destination_config_exists = any(
+        os.path.exists(filename)
+        for filename in destination_config_filenames
+    )
+
+    if os.path.exists(source_config_filename) and not destination_config_exists:
         raise LegacyConfigurationNotUpgraded()
 
 
diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py
index f1cc43a6a..033434153 100644
--- a/borgmatic/tests/integration/commands/test_borgmatic.py
+++ b/borgmatic/tests/integration/commands/test_borgmatic.py
@@ -9,23 +9,30 @@ from borgmatic.commands import borgmatic as module
 def test_parse_arguments_with_no_arguments_uses_defaults():
     parser = module.parse_arguments()
 
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
     assert parser.excludes_filename == None
     assert parser.verbosity is None
 
 
-def test_parse_arguments_with_filename_arguments_overrides_defaults():
+def test_parse_arguments_with_path_arguments_overrides_defaults():
     parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
 
-    assert parser.config_filename == 'myconfig'
+    assert parser.config_paths == ['myconfig']
     assert parser.excludes_filename == 'myexcludes'
     assert parser.verbosity is None
 
 
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    assert parser.config_paths == ['myconfig', 'otherconfig']
+    assert parser.verbosity is None
+
+
 def test_parse_arguments_with_verbosity_flag_overrides_default():
     parser = module.parse_arguments('--verbosity', '1')
 
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
     assert parser.excludes_filename == None
     assert parser.verbosity == 1
 
diff --git a/borgmatic/tests/unit/config/test_collect.py b/borgmatic/tests/unit/config/test_collect.py
new file mode 100644
index 000000000..2adee63e0
--- /dev/null
+++ b/borgmatic/tests/unit/config/test_collect.py
@@ -0,0 +1,58 @@
+from flexmock import flexmock
+
+from borgmatic.config import collect as module
+
+
+def test_collect_config_filenames_collects_given_files():
+    config_paths = ('config.yaml', 'other.yaml')
+    flexmock(module.os.path).should_receive('isdir').and_return(False)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == config_paths
+
+
+def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories():
+    config_paths = ('config.yaml', '/etc/borgmatic.d')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').and_return(True)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
+    flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar', 'baz.yaml'])
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == (
+        'config.yaml',
+        '/etc/borgmatic.d/foo.yaml',
+        '/etc/borgmatic.d/baz.yaml',
+    )
+
+
+def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist():
+    config_paths = ('config.yaml', '/etc/borgmatic.d')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
+    mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == ('config.yaml',)
+
+
+def test_collect_config_filenames_includes_directory_if_it_does_not_exist():
+    config_paths = ('config.yaml', '/my/directory')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
+    mock_path.should_receive('exists').with_args('/my/directory').and_return(False)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/my/directory').and_return(True)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == config_paths
diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py
index 58693ab2c..995fc8a5b 100644
--- a/borgmatic/tests/unit/config/test_convert.py
+++ b/borgmatic/tests/unit/config/test_convert.py
@@ -79,30 +79,34 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
 def test_guard_configuration_upgraded_raises_when_only_source_config_present():
     flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
     flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
 
     with pytest.raises(module.LegacyConfigurationNotUpgraded):
-        module.guard_configuration_upgraded('config', 'config.yaml')
+        module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
 
 
 def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present():
     flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
-    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
 
-    module.guard_configuration_upgraded('config', 'config.yaml')
+    module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
 
 
 def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present():
     flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
-    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
 
-    module.guard_configuration_upgraded('config', 'config.yaml')
+    module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
 
 
 def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present():
     flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
     flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
 
-    module.guard_configuration_upgraded('config', 'config.yaml')
+    module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
 
 
 def test_guard_excludes_filename_omitted_raises_when_filename_provided():
diff --git a/setup.py b/setup.py
index 6d4cc7c36..143736bcc 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.2'
+VERSION = '1.1.3.dev0'
 
 
 setup(

From 94aaf4554fca77d4151e069065086108112f8145 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 25 Jul 2017 21:21:47 -0700
Subject: [PATCH 161/189] Releasing.

---
 NEWS     | 2 +-
 setup.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 3bc88855e..f73b4b8f0 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
-1.1.3.dev0
+1.1.3
 
  * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
  * Fix for generate-borgmatic-config writing config with invalid one_file_system value.
diff --git a/setup.py b/setup.py
index 143736bcc..7831a3ec8 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.3.dev0'
+VERSION = '1.1.3'
 
 
 setup(

From 10404143c66c02e3bf8351315de0345c20f66332 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Tue, 25 Jul 2017 21:21:50 -0700
Subject: [PATCH 162/189] Added tag 1.1.3 for changeset 3f838f661546

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 24e679862..95e8f524d 100644
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
 5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0
 7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1
 f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2
+3f838f661546e04529b453aa443529b432afc243 1.1.3

From a2e8abc53794e162bf338629e955227bdf9fe6cc Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 28 Jul 2017 22:02:18 -0700
Subject: [PATCH 163/189] #17: Added command-line flags for performing a
 borgmatic run with only pruning, creating, or checking enabled.

---
 NEWS                                          |  6 ++
 README.md                                     | 22 ++++++-
 borgmatic/commands/borgmatic.py               | 60 +++++++++++++++----
 .../integration/commands/test_borgmatic.py    | 24 ++++++++
 setup.py                                      |  2 +-
 5 files changed, 99 insertions(+), 15 deletions(-)

diff --git a/NEWS b/NEWS
index f73b4b8f0..c5a81627e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,9 @@
+1.1.4
+
+ * #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or
+   checking enabled. This supports use cases like running consistency checks from a different cron
+   job with a different frequency, or running pruning with a different verbosity level.
+
 1.1.3
 
  * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
diff --git a/README.md b/README.md
index 8ea9a8929..c15e826a2 100644
--- a/README.md
+++ b/README.md
@@ -83,7 +83,7 @@ to remove anything you don't need.
 ### Multiple configuration files
 
 A more advanced usage is to create multiple separate configuration files and
-place each one in a /etc/borgmatic.d directory. For instance:
+place each one in an /etc/borgmatic.d directory. For instance:
 
     sudo mkdir /etc/borgmatic.d
     sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
@@ -175,6 +175,12 @@ arguments:
 This will also prune any old backups as per the configured retention policy,
 and check backups for consistency problems due to things like file damage.
 
+If you'd like to see the available command-line arguments, view the help:
+
+    borgmatic --help
+
+### Verbosity
+
 By default, the backup will proceed silently except in the case of errors. But
 if you'd like to to get additional information about the progress of the
 backup as it proceeds, use the verbosity option:
@@ -185,9 +191,19 @@ Or, for even more progress spew:
 
     borgmatic --verbosity 2
 
-If you'd like to see the available command-line arguments, view the help:
+### À la carte
 
-    borgmatic --help
+If you want to run borgmatic with only pruning, creating, or checking enabled,
+the following optional flags are available:
+
+    borgmatic --prune
+    borgmatic --create
+    borgmatic --check
+
+You can run with only one of these flags provided, or you can mix and match
+any number of them. This supports use cases like running consistency checks
+from a different cron job with a different frequency, or running pruning with
+a different verbosity level.
 
 
 ## Autopilot
diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index a26c30cd6..c5c7a28d7 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -18,7 +18,14 @@ def parse_arguments(*arguments):
     Given command-line arguments with which this script was invoked, parse the arguments and return
     them as an ArgumentParser instance.
     '''
-    parser = ArgumentParser()
+    parser = ArgumentParser(
+        description=
+            '''
+            A simple wrapper script for the Borg backup software that creates and prunes backups.
+            If none of the --prune, --create, or --check options are given, then borgmatic defaults
+            to all three: prune, create, and check archives.
+            '''
+    )
     parser.add_argument(
         '-c', '--config',
         nargs='+',
@@ -29,7 +36,25 @@ def parse_arguments(*arguments):
     parser.add_argument(
         '--excludes',
         dest='excludes_filename',
-        help='Excludes filename, deprecated in favor of exclude_patterns within configuration',
+        help='Deprecated in favor of exclude_patterns within configuration',
+    )
+    parser.add_argument(
+        '-p', '--prune',
+        dest='prune',
+        action='store_true',
+        help='Prune archives according to the retention policy',
+    )
+    parser.add_argument(
+        '-C', '--create',
+        dest='create',
+        action='store_true',
+        help='Create archives (actually perform backups)',
+    )
+    parser.add_argument(
+        '-k', '--check',
+        dest='check',
+        action='store_true',
+        help='Check archives for consistency',
     )
     parser.add_argument(
         '-v', '--verbosity',
@@ -37,7 +62,17 @@ def parse_arguments(*arguments):
         help='Display verbose progress (1 for some, 2 for lots)',
     )
 
-    return parser.parse_args(arguments)
+    args = parser.parse_args(arguments)
+
+    # If any of the three action flags in the given parse arguments have been explicitly requested,
+    # leave them as-is. Otherwise, assume defaults: Mutate the given arguments to enable all the
+    # actions.
+    if not args.prune and not args.create and not args.check:
+        args.prune = True
+        args.create = True
+        args.check = True
+
+    return args
 
 
 def main():  # pragma: no cover
@@ -60,14 +95,17 @@ def main():  # pragma: no cover
             borg.initialize(storage)
 
             for repository in location['repositories']:
-                borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
-                borg.create_archive(
-                    args.verbosity,
-                    repository,
-                    location,
-                    storage,
-                )
-                borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
+                if args.prune:
+                    borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+                if args.create:
+                    borg.create_archive(
+                        args.verbosity,
+                        repository,
+                        location,
+                        storage,
+                    )
+                if args.check:
+                    borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py
index 033434153..2b82ecfb1 100644
--- a/borgmatic/tests/integration/commands/test_borgmatic.py
+++ b/borgmatic/tests/integration/commands/test_borgmatic.py
@@ -37,6 +37,30 @@ def test_parse_arguments_with_verbosity_flag_overrides_default():
     assert parser.verbosity == 1
 
 
+def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
+    parser = module.parse_arguments()
+
+    assert parser.prune is True
+    assert parser.create is True
+    assert parser.check is True
+
+
+def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
+    parser = module.parse_arguments('--prune')
+
+    assert parser.prune is True
+    assert parser.create is False
+    assert parser.check is False
+
+
+def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
+    parser = module.parse_arguments('--create', '--check')
+
+    assert parser.prune is False
+    assert parser.create is True
+    assert parser.check is True
+
+
 def test_parse_arguments_with_invalid_arguments_exits():
     with pytest.raises(SystemExit):
         module.parse_arguments('--posix-me-harder')
diff --git a/setup.py b/setup.py
index 7831a3ec8..d4f587b3e 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.3'
+VERSION = '1.1.4'
 
 
 setup(

From ae15e0f4047180bbbde347def932ee96fa2b51fb Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 28 Jul 2017 22:02:43 -0700
Subject: [PATCH 164/189] Added tag 1.1.4 for changeset 3d605962d891

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 95e8f524d..50e938ab3 100644
--- a/.hgtags
+++ b/.hgtags
@@ -34,3 +34,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
 7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1
 f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2
 3f838f661546e04529b453aa443529b432afc243 1.1.3
+3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4

From 0f44fbedf41301fee347413a223aed1547a70288 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 28 Jul 2017 22:36:16 -0700
Subject: [PATCH 165/189] Getting logo to show up on GitHub.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c15e826a2..09c19a1be 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 title: borgmatic
 
-<img src="static/borgmatic.svg" alt="borgmatic logo" style="width: 8em; float: right; padding-left: 1em;" />
+<img src="https://cdn.rawgit.com/witten/borgmatic/master/static/borgmatic.svg" width="150px" style="float: right; padding-left: 1em;">
 
 ## Overview
 

From 23679a6eddec21e7e0e6eb2c4a5e9bf6ed3d513b Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 29 Jul 2017 16:05:11 -0700
Subject: [PATCH 166/189] Removing Pelican-specific title metadata out of
 README markdown.

---
 README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.md b/README.md
index 09c19a1be..22b1da7db 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,3 @@
-title: borgmatic
-
 <img src="https://cdn.rawgit.com/witten/borgmatic/master/static/borgmatic.svg" width="150px" style="float: right; padding-left: 1em;">
 
 ## Overview

From 9bea7ae5edac546ccb014141b1e30f4fe7cb383d Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 30 Jul 2017 11:16:26 -0700
Subject: [PATCH 167/189] #34: New "extract" consistency check that performs a
 dry-run extraction of the most recent archive.

---
 NEWS                              |   5 +
 README.md                         |   6 ++
 borgmatic/borg.py                 |  66 ++++++++++---
 borgmatic/config/schema.yaml      |  16 ++--
 borgmatic/tests/unit/test_borg.py | 148 +++++++++++++++++++++++++++---
 setup.py                          |   2 +-
 6 files changed, 211 insertions(+), 32 deletions(-)

diff --git a/NEWS b/NEWS
index c5a81627e..6222fea6c 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+1.1.5
+
+ * #34: New "extract" consistency check that performs a dry-run extraction of the most recent
+   archive.
+
 1.1.4
 
  * #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or
diff --git a/README.md b/README.md
index 22b1da7db..057388bf7 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,12 @@ default). You should edit the file to suit your needs, as the values are just
 representative. All fields are optional except where indicated, so feel free
 to remove anything you don't need.
 
+You can also have a look at the [full configuration
+schema](https://torsion.org/hg/borgmatic/file/tip/borgmatic/config/schema.yaml)
+for the authoritative set of all configuration options. This is handy if
+borgmatic has added new options since you originally created your
+configuration file.
+
 
 ### Multiple configuration files
 
diff --git a/borgmatic/borg.py b/borgmatic/borg.py
index 8c4c73086..fff6e7ef3 100644
--- a/borgmatic/borg.py
+++ b/borgmatic/borg.py
@@ -3,6 +3,7 @@ import glob
 import itertools
 import os
 import platform
+import sys
 import re
 import subprocess
 import tempfile
@@ -43,7 +44,7 @@ def create_archive(
 ):
     '''
     Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
-    repository path, a list of exclude patterns, and a command to run, create an attic archive.
+    repository path, a list of exclude patterns, and a command to run, create a Borg archive.
     '''
     sources = tuple(
         itertools.chain.from_iterable(
@@ -68,8 +69,8 @@ def create_archive(
 
     full_command = (
         command, 'create',
-        '{repo}::{hostname}-{timestamp}'.format(
-            repo=repository,
+        '{repository}::{hostname}-{timestamp}'.format(
+            repository=repository,
             hostname=platform.node(),
             timestamp=datetime.now().isoformat(),
         ),
@@ -104,7 +105,7 @@ def _make_prune_flags(retention_config):
 def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None):
     '''
     Given a verbosity flag, a local or remote repository path, a retention config dict, and a
-    command to run, prune attic archives according the the retention policy specified in that
+    command to run, prune Borg archives according the the retention policy specified in that
     configuration.
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
@@ -170,33 +171,72 @@ def _make_check_flags(checks, check_last=None):
 
     return tuple(
         '--{}-only'.format(check) for check in checks
+        if check in DEFAULT_CHECKS
     ) + last_flag
 
 
 def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None):
     '''
     Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
-    command to run, check the contained attic archives for consistency.
+    command to run, check the contained Borg archives for consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
     checks = _parse_checks(consistency_config)
     check_last = consistency_config.get('check_last', None)
-    if not checks:
-        return
 
+    if set(checks).intersection(set(DEFAULT_CHECKS)):
+        remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+        verbosity_flags = {
+            VERBOSITY_SOME: ('--info',),
+            VERBOSITY_LOTS: ('--debug',),
+        }.get(verbosity, ())
+
+        full_command = (
+            command, 'check',
+            repository,
+        ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
+
+        # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
+        stdout = None if verbosity_flags else open(os.devnull, 'w')
+
+        subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
+
+    if 'extract' in checks:
+        extract_last_archive_dry_run(verbosity, repository, command, remote_path)
+
+
+def extract_last_archive_dry_run(verbosity, repository, command=COMMAND, remote_path=None):
+    '''
+    Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
+    the dry-run.
+    '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--info',),
         VERBOSITY_LOTS: ('--debug',),
     }.get(verbosity, ())
 
-    full_command = (
-        command, 'check',
+    full_list_command = (
+        command, 'list',
+        '--short',
         repository,
-    ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
+    ) + remote_path_flags + verbosity_flags
 
-    # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
-    stdout = None if verbosity_flags else open(os.devnull, 'w')
+    list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
 
-    subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
+    last_archive_name = list_output.strip().split('\n')[-1]
+    if not last_archive_name:
+        return
+
+    list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
+    full_extract_command = (
+        command, 'extract',
+        '--dry-run',
+        '{repository}::{last_archive_name}'.format(
+            repository=repository,
+            last_archive_name=last_archive_name,
+        ),
+    ) + remote_path_flags + verbosity_flags + list_flag
+
+    subprocess.check_call(full_extract_command)
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index 07e245d6f..567ec7329 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -106,21 +106,25 @@ map:
     consistency:
         desc: |
             Consistency checks to run after backups. See
-            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details.
         map:
             checks:
                 seq:
                     - type: str
-                      enum: ['repository', 'archives', 'disabled']
+                      enum: ['repository', 'archives', 'extract', 'disabled']
                       unique: true
                 desc: |
-                    List of consistency checks to run: "repository", "archives", or both. Defaults
-                    to both. Set to "disabled" to disable all consistency checks. See
-                    https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
+                    List of one or more consistency checks to run: "repository", "archives", and/or
+                    "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
+                    all consistency checks. "repository" checks the consistency of the repository,
+                    "archive" checks all of the archives, and "extract" does an extraction dry-run
+                    of just the most recent archive.
                 example:
                     - repository
                     - archives
             check_last:
                 type: int
-                desc: Restrict the number of checked archives to the last n.
+                desc: Restrict the number of checked archives to the last n. Applies only to the
+                      "archives" check.
                 example: 3
diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py
index 4963ab539..1cba04078 100644
--- a/borgmatic/tests/unit/test_borg.py
+++ b/borgmatic/tests/unit/test_borg.py
@@ -4,6 +4,7 @@ import sys
 import os
 
 from flexmock import flexmock
+import pytest
 
 from borgmatic import borg as module
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
@@ -46,17 +47,23 @@ def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
 
 
 def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(STDOUT=STDOUT)
+    subprocess = flexmock(module.subprocess)
     subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
     flexmock(module).subprocess = subprocess
 
 
 def insert_subprocess_never():
-    subprocess = flexmock()
+    subprocess = flexmock(module.subprocess)
     subprocess.should_receive('check_call').never()
     flexmock(module).subprocess = subprocess
 
 
+def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once()
+    flexmock(module).subprocess = subprocess
+
+
 def insert_platform_mock():
     flexmock(module.platform).should_receive('node').and_return('host')
 
@@ -395,9 +402,15 @@ def test_parse_checks_with_disabled_returns_no_checks():
 
 
 def test_make_check_flags_with_checks_returns_flags():
-    flags = module._make_check_flags(('foo', 'bar'))
+    flags = module._make_check_flags(('repository',))
 
-    assert flags == ('--foo-only', '--bar-only')
+    assert flags == ('--repository-only',)
+
+
+def test_make_check_flags_with_extract_check_does_not_make_extract_flag():
+    flags = module._make_check_flags(('extract',))
+
+    assert flags == ()
 
 
 def test_make_check_flags_with_default_checks_returns_no_flags():
@@ -407,19 +420,27 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
 
 
 def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
-    flags = module._make_check_flags(('foo', 'bar'), check_last=3)
+    flags = module._make_check_flags(('repository',), check_last=3)
 
-    assert flags == ('--foo-only', '--bar-only', '--last', '3')
+    assert flags == ('--repository-only', '--last', '3')
 
 
-def test_make_check_flags_with_last_returns_last_flag():
+def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
     flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
 
     assert flags == ('--last', '3')
 
 
-def test_check_archives_should_call_borg_with_parameters():
-    checks = flexmock()
+@pytest.mark.parametrize(
+    'checks',
+    (
+        ('repository',),
+        ('archives',),
+        ('repository', 'archives'),
+        ('repository', 'archives', 'other'),
+    ),
+)
+def test_check_archives_should_call_borg_with_parameters(checks):
     check_last = flexmock()
     consistency_config = flexmock().should_receive('get').and_return(check_last).mock
     flexmock(module).should_receive('_parse_checks').and_return(checks)
@@ -442,9 +463,27 @@ def test_check_archives_should_call_borg_with_parameters():
     )
 
 
+def test_check_archives_with_extract_check_should_call_extract_only():
+    checks = ('extract',)
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').never()
+    flexmock(module).should_receive('extract_last_archive_dry_run').once()
+    insert_subprocess_never()
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        consistency_config=consistency_config,
+        command='borg',
+    )
+
+
 def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
+    checks = ('repository',)
     consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
         ('borg', 'check', 'repo', '--info'),
@@ -462,8 +501,9 @@ def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter
 
 
 def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    checks = ('repository',)
     consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
         ('borg', 'check', 'repo', '--debug'),
@@ -494,7 +534,7 @@ def test_check_archives_without_any_checks_should_bail():
 
 
 def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
-    checks = flexmock()
+    checks = ('repository',)
     check_last = flexmock()
     consistency_config = flexmock().should_receive('get').and_return(check_last).mock
     flexmock(module).should_receive('_parse_checks').and_return(checks)
@@ -516,3 +556,87 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
         command='borg',
         remote_path='borg1',
     )
+
+
+def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        command='borg',
+    )
+
+
+def test_extract_last_archive_dry_run_without_any_archives_should_bail():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo'),
+        result='\n'.encode('utf-8'),
+    )
+    insert_subprocess_never()
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        command='borg',
+    )
+
+
+def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--info'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--info'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=VERBOSITY_SOME,
+        repository='repo',
+        command='borg',
+    )
+
+
+def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--debug'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=VERBOSITY_LOTS,
+        repository='repo',
+        command='borg',
+    )
+
+
+def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        command='borg',
+        remote_path='borg1',
+    )
diff --git a/setup.py b/setup.py
index d4f587b3e..2ff96905a 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.4'
+VERSION = '1.1.5'
 
 
 setup(

From 77d3c66fb98ab72cd27000b5fcf8e81ae56b8125 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 30 Jul 2017 11:16:41 -0700
Subject: [PATCH 168/189] Added tag 1.1.5 for changeset 64ca13bfe050

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 50e938ab3..50ee67dcf 100644
--- a/.hgtags
+++ b/.hgtags
@@ -35,3 +35,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
 f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2
 3f838f661546e04529b453aa443529b432afc243 1.1.3
 3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4
+64ca13bfe050f656b44ed2eb1c3db045bfddd133 1.1.5

From aa04473521f0ce9c0bb4469cd60daea501a5fb6c Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 5 Aug 2017 16:21:39 -0700
Subject: [PATCH 169/189] Split out Borg integration code into multiple files,
 as it was getting kind of hairy all in one.

---
 borgmatic/borg.py                         | 242 --------
 borgmatic/borg/__init__.py                |   0
 borgmatic/borg/check.py                   |  85 +++
 borgmatic/borg/create.py                  |  72 +++
 borgmatic/borg/extract.py                 |  40 ++
 borgmatic/borg/prune.py                   |  48 ++
 borgmatic/tests/unit/borg/__init__.py     |   0
 borgmatic/tests/unit/borg/test_check.py   | 185 +++++++
 borgmatic/tests/unit/borg/test_create.py  | 264 +++++++++
 borgmatic/tests/unit/borg/test_extract.py | 100 ++++
 borgmatic/tests/unit/borg/test_prune.py   |  93 ++++
 borgmatic/tests/unit/test_borg.py         | 642 ----------------------
 12 files changed, 887 insertions(+), 884 deletions(-)
 delete mode 100644 borgmatic/borg.py
 create mode 100644 borgmatic/borg/__init__.py
 create mode 100644 borgmatic/borg/check.py
 create mode 100644 borgmatic/borg/create.py
 create mode 100644 borgmatic/borg/extract.py
 create mode 100644 borgmatic/borg/prune.py
 create mode 100644 borgmatic/tests/unit/borg/__init__.py
 create mode 100644 borgmatic/tests/unit/borg/test_check.py
 create mode 100644 borgmatic/tests/unit/borg/test_create.py
 create mode 100644 borgmatic/tests/unit/borg/test_extract.py
 create mode 100644 borgmatic/tests/unit/borg/test_prune.py
 delete mode 100644 borgmatic/tests/unit/test_borg.py

diff --git a/borgmatic/borg.py b/borgmatic/borg.py
deleted file mode 100644
index fff6e7ef3..000000000
--- a/borgmatic/borg.py
+++ /dev/null
@@ -1,242 +0,0 @@
-from datetime import datetime
-import glob
-import itertools
-import os
-import platform
-import sys
-import re
-import subprocess
-import tempfile
-
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-# Integration with Borg for actually handling backups.
-
-
-COMMAND = 'borg'
-
-
-def initialize(storage_config, command=COMMAND):
-    passphrase = storage_config.get('encryption_passphrase')
-
-    if passphrase:
-        os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase
-
-
-def _write_exclude_file(exclude_patterns=None):
-    '''
-    Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
-    None if no patterns are provided.
-    '''
-    if not exclude_patterns:
-        return None
-
-    exclude_file = tempfile.NamedTemporaryFile('w')
-    exclude_file.write('\n'.join(exclude_patterns))
-    exclude_file.flush()
-
-    return exclude_file
-
-
-def create_archive(
-    verbosity, repository, location_config, storage_config, command=COMMAND,
-):
-    '''
-    Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
-    repository path, a list of exclude patterns, and a command to run, create a Borg archive.
-    '''
-    sources = tuple(
-        itertools.chain.from_iterable(
-            glob.glob(directory) or [directory]
-            for directory in location_config['source_directories']
-        )
-    )
-
-    exclude_file = _write_exclude_file(location_config.get('exclude_patterns'))
-    exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
-    compression = storage_config.get('compression', None)
-    compression_flags = ('--compression', compression) if compression else ()
-    umask = storage_config.get('umask', None)
-    umask_flags = ('--umask', str(umask)) if umask else ()
-    one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
-    remote_path = location_config.get('remote_path')
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--list', '--stats'),
-    }.get(verbosity, ())
-
-    full_command = (
-        command, 'create',
-        '{repository}::{hostname}-{timestamp}'.format(
-            repository=repository,
-            hostname=platform.node(),
-            timestamp=datetime.now().isoformat(),
-        ),
-    ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
-        remote_path_flags + umask_flags + verbosity_flags
-
-    subprocess.check_call(full_command)
-
-
-def _make_prune_flags(retention_config):
-    '''
-    Given a retention config dict mapping from option name to value, tranform it into an iterable of
-    command-line name-value flag pairs.
-
-    For example, given a retention config of:
-
-        {'keep_weekly': 4, 'keep_monthly': 6}
-
-    This will be returned as an iterable of:
-
-        (
-            ('--keep-weekly', '4'),
-            ('--keep-monthly', '6'),
-        )
-    '''
-    return (
-        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
-        for option_name, value in retention_config.items()
-    )
-
-
-def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None):
-    '''
-    Given a verbosity flag, a local or remote repository path, a retention config dict, and a
-    command to run, prune Borg archives according the the retention policy specified in that
-    configuration.
-    '''
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--stats'),
-    }.get(verbosity, ())
-
-    full_command = (
-        command, 'prune',
-        repository,
-    ) + tuple(
-        element
-        for pair in _make_prune_flags(retention_config)
-        for element in pair
-    ) + remote_path_flags + verbosity_flags
-
-    subprocess.check_call(full_command)
-
-
-DEFAULT_CHECKS = ('repository', 'archives')
-
-
-def _parse_checks(consistency_config):
-    '''
-    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
-
-    For example, given a retention config of:
-
-        {'checks': ['repository', 'archives']}
-
-    This will be returned as:
-
-        ('repository', 'archives')
-
-    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
-    "disabled", return an empty tuple, meaning that no checks should be run.
-    '''
-    checks = consistency_config.get('checks', [])
-    if checks == ['disabled']:
-        return ()
-
-    return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
-
-
-def _make_check_flags(checks, check_last=None):
-    '''
-    Given a parsed sequence of checks, transform it into tuple of command-line flags.
-
-    For example, given parsed checks of:
-
-        ('repository',)
-
-    This will be returned as:
-
-        ('--repository-only',)
-
-    Additionally, if a check_last value is given, a "--last" flag will be added.
-    '''
-    last_flag = ('--last', str(check_last)) if check_last else ()
-    if checks == DEFAULT_CHECKS:
-        return last_flag
-
-    return tuple(
-        '--{}-only'.format(check) for check in checks
-        if check in DEFAULT_CHECKS
-    ) + last_flag
-
-
-def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None):
-    '''
-    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
-    command to run, check the contained Borg archives for consistency.
-
-    If there are no consistency checks to run, skip running them.
-    '''
-    checks = _parse_checks(consistency_config)
-    check_last = consistency_config.get('check_last', None)
-
-    if set(checks).intersection(set(DEFAULT_CHECKS)):
-        remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-        verbosity_flags = {
-            VERBOSITY_SOME: ('--info',),
-            VERBOSITY_LOTS: ('--debug',),
-        }.get(verbosity, ())
-
-        full_command = (
-            command, 'check',
-            repository,
-        ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
-
-        # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
-        stdout = None if verbosity_flags else open(os.devnull, 'w')
-
-        subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
-
-    if 'extract' in checks:
-        extract_last_archive_dry_run(verbosity, repository, command, remote_path)
-
-
-def extract_last_archive_dry_run(verbosity, repository, command=COMMAND, remote_path=None):
-    '''
-    Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
-    the dry-run.
-    '''
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info',),
-        VERBOSITY_LOTS: ('--debug',),
-    }.get(verbosity, ())
-
-    full_list_command = (
-        command, 'list',
-        '--short',
-        repository,
-    ) + remote_path_flags + verbosity_flags
-
-    list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
-
-    last_archive_name = list_output.strip().split('\n')[-1]
-    if not last_archive_name:
-        return
-
-    list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
-    full_extract_command = (
-        command, 'extract',
-        '--dry-run',
-        '{repository}::{last_archive_name}'.format(
-            repository=repository,
-            last_archive_name=last_archive_name,
-        ),
-    ) + remote_path_flags + verbosity_flags + list_flag
-
-    subprocess.check_call(full_extract_command)
diff --git a/borgmatic/borg/__init__.py b/borgmatic/borg/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py
new file mode 100644
index 000000000..49805547f
--- /dev/null
+++ b/borgmatic/borg/check.py
@@ -0,0 +1,85 @@
+import os
+import subprocess
+
+from borgmatic.borg import extract
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+DEFAULT_CHECKS = ('repository', 'archives')
+
+
+def _parse_checks(consistency_config):
+    '''
+    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
+
+    For example, given a retention config of:
+
+        {'checks': ['repository', 'archives']}
+
+    This will be returned as:
+
+        ('repository', 'archives')
+
+    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
+    "disabled", return an empty tuple, meaning that no checks should be run.
+    '''
+    checks = consistency_config.get('checks', [])
+    if checks == ['disabled']:
+        return ()
+
+    return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
+
+
+def _make_check_flags(checks, check_last=None):
+    '''
+    Given a parsed sequence of checks, transform it into tuple of command-line flags.
+
+    For example, given parsed checks of:
+
+        ('repository',)
+
+    This will be returned as:
+
+        ('--repository-only',)
+
+    Additionally, if a check_last value is given, a "--last" flag will be added.
+    '''
+    last_flag = ('--last', str(check_last)) if check_last else ()
+    if checks == DEFAULT_CHECKS:
+        return last_flag
+
+    return tuple(
+        '--{}-only'.format(check) for check in checks
+        if check in DEFAULT_CHECKS
+    ) + last_flag
+
+
+def check_archives(verbosity, repository, consistency_config, remote_path=None):
+    '''
+    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
+    command to run, check the contained Borg archives for consistency.
+
+    If there are no consistency checks to run, skip running them.
+    '''
+    checks = _parse_checks(consistency_config)
+    check_last = consistency_config.get('check_last', None)
+
+    if set(checks).intersection(set(DEFAULT_CHECKS)):
+        remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+        verbosity_flags = {
+            VERBOSITY_SOME: ('--info',),
+            VERBOSITY_LOTS: ('--debug',),
+        }.get(verbosity, ())
+
+        full_command = (
+            'borg', 'check',
+            repository,
+        ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
+
+        # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
+        stdout = None if verbosity_flags else open(os.devnull, 'w')
+
+        subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
+
+    if 'extract' in checks:
+        extract.extract_last_archive_dry_run(verbosity, repository, remote_path)
diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py
new file mode 100644
index 000000000..21af701c8
--- /dev/null
+++ b/borgmatic/borg/create.py
@@ -0,0 +1,72 @@
+from datetime import datetime
+import glob
+import itertools
+import os
+import platform
+import subprocess
+import tempfile
+
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def initialize(storage_config):
+    passphrase = storage_config.get('encryption_passphrase')
+
+    if passphrase:
+        os.environ['BORG_PASSPHRASE'] = passphrase
+
+
+def _write_exclude_file(exclude_patterns=None):
+    '''
+    Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
+    None if no patterns are provided.
+    '''
+    if not exclude_patterns:
+        return None
+
+    exclude_file = tempfile.NamedTemporaryFile('w')
+    exclude_file.write('\n'.join(exclude_patterns))
+    exclude_file.flush()
+
+    return exclude_file
+
+
+def create_archive(
+    verbosity, repository, location_config, storage_config,
+):
+    '''
+    Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
+    repository path, a list of exclude patterns, create a Borg archive.
+    '''
+    sources = tuple(
+        itertools.chain.from_iterable(
+            glob.glob(directory) or [directory]
+            for directory in location_config['source_directories']
+        )
+    )
+
+    exclude_file = _write_exclude_file(location_config.get('exclude_patterns'))
+    exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
+    compression = storage_config.get('compression', None)
+    compression_flags = ('--compression', compression) if compression else ()
+    umask = storage_config.get('umask', None)
+    umask_flags = ('--umask', str(umask)) if umask else ()
+    one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
+    remote_path = location_config.get('remote_path')
+    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--info', '--stats',),
+        VERBOSITY_LOTS: ('--debug', '--list', '--stats'),
+    }.get(verbosity, ())
+
+    full_command = (
+        'borg', 'create',
+        '{repository}::{hostname}-{timestamp}'.format(
+            repository=repository,
+            hostname=platform.node(),
+            timestamp=datetime.now().isoformat(),
+        ),
+    ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
+        remote_path_flags + umask_flags + verbosity_flags
+
+    subprocess.check_call(full_command)
diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py
new file mode 100644
index 000000000..fde1ac591
--- /dev/null
+++ b/borgmatic/borg/extract.py
@@ -0,0 +1,40 @@
+import sys
+import subprocess
+
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def extract_last_archive_dry_run(verbosity, repository, remote_path=None):
+    '''
+    Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
+    the dry-run.
+    '''
+    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--info',),
+        VERBOSITY_LOTS: ('--debug',),
+    }.get(verbosity, ())
+
+    full_list_command = (
+        'borg', 'list',
+        '--short',
+        repository,
+    ) + remote_path_flags + verbosity_flags
+
+    list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
+
+    last_archive_name = list_output.strip().split('\n')[-1]
+    if not last_archive_name:
+        return
+
+    list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
+    full_extract_command = (
+        'borg', 'extract',
+        '--dry-run',
+        '{repository}::{last_archive_name}'.format(
+            repository=repository,
+            last_archive_name=last_archive_name,
+        ),
+    ) + remote_path_flags + verbosity_flags + list_flag
+
+    subprocess.check_call(full_extract_command)
diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py
new file mode 100644
index 000000000..8f52cb4c0
--- /dev/null
+++ b/borgmatic/borg/prune.py
@@ -0,0 +1,48 @@
+import subprocess
+
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def _make_prune_flags(retention_config):
+    '''
+    Given a retention config dict mapping from option name to value, tranform it into an iterable of
+    command-line name-value flag pairs.
+
+    For example, given a retention config of:
+
+        {'keep_weekly': 4, 'keep_monthly': 6}
+
+    This will be returned as an iterable of:
+
+        (
+            ('--keep-weekly', '4'),
+            ('--keep-monthly', '6'),
+        )
+    '''
+    return (
+        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
+        for option_name, value in retention_config.items()
+    )
+
+
+def prune_archives(verbosity, repository, retention_config, remote_path=None):
+    '''
+    Given a verbosity flag, a local or remote repository path, a retention config dict, prune Borg
+    archives according the the retention policy specified in that configuration.
+    '''
+    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--info', '--stats',),
+        VERBOSITY_LOTS: ('--debug', '--stats'),
+    }.get(verbosity, ())
+
+    full_command = (
+        'borg', 'prune',
+        repository,
+    ) + tuple(
+        element
+        for pair in _make_prune_flags(retention_config)
+        for element in pair
+    ) + remote_path_flags + verbosity_flags
+
+    subprocess.check_call(full_command)
diff --git a/borgmatic/tests/unit/borg/__init__.py b/borgmatic/tests/unit/borg/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/borgmatic/tests/unit/borg/test_check.py b/borgmatic/tests/unit/borg/test_check.py
new file mode 100644
index 000000000..2b66f0960
--- /dev/null
+++ b/borgmatic/tests/unit/borg/test_check.py
@@ -0,0 +1,185 @@
+from subprocess import STDOUT
+import sys
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic.borg import check as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+
+
+def insert_subprocess_never():
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').never()
+
+
+def test_parse_checks_returns_them_as_tuple():
+    checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']})
+
+    assert checks == ('foo', 'bar')
+
+
+def test_parse_checks_with_missing_value_returns_defaults():
+    checks = module._parse_checks({})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_blank_value_returns_defaults():
+    checks = module._parse_checks({'checks': []})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_disabled_returns_no_checks():
+    checks = module._parse_checks({'checks': ['disabled']})
+
+    assert checks == ()
+
+
+def test_make_check_flags_with_checks_returns_flags():
+    flags = module._make_check_flags(('repository',))
+
+    assert flags == ('--repository-only',)
+
+
+def test_make_check_flags_with_extract_check_does_not_make_extract_flag():
+    flags = module._make_check_flags(('extract',))
+
+    assert flags == ()
+
+
+def test_make_check_flags_with_default_checks_returns_no_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS)
+
+    assert flags == ()
+
+
+def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
+    flags = module._make_check_flags(('repository',), check_last=3)
+
+    assert flags == ('--repository-only', '--last', '3')
+
+
+def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
+
+    assert flags == ('--last', '3')
+
+
+@pytest.mark.parametrize(
+    'checks',
+    (
+        ('repository',),
+        ('archives',),
+        ('repository', 'archives'),
+        ('repository', 'archives', 'other'),
+    ),
+)
+def test_check_archives_should_call_borg_with_parameters(checks):
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_extract_check_should_call_extract_only():
+    checks = ('extract',)
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').never()
+    flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
+    insert_subprocess_never()
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
+    checks = ('repository',)
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--info'),
+        stdout=None, stderr=STDOUT,
+    )
+
+    module.check_archives(
+        verbosity=VERBOSITY_SOME,
+        repository='repo',
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    checks = ('repository',)
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--debug'),
+        stdout=None, stderr=STDOUT,
+    )
+
+    module.check_archives(
+        verbosity=VERBOSITY_LOTS,
+        repository='repo',
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_without_any_checks_should_bail():
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
+    flexmock(module).should_receive('_parse_checks').and_return(())
+    insert_subprocess_never()
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
+    checks = ('repository',)
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--remote-path', 'borg1'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        consistency_config=consistency_config,
+        remote_path='borg1',
+    )
diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py
new file mode 100644
index 000000000..3b4d2689c
--- /dev/null
+++ b/borgmatic/tests/unit/borg/test_create.py
@@ -0,0 +1,264 @@
+import os
+
+from flexmock import flexmock
+
+from borgmatic.borg import create as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def test_initialize_with_passphrase_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({'encryption_passphrase': 'pass'})
+        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_without_passphrase_should_not_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({})
+        assert os.environ.get('BORG_PASSPHRASE') == None
+    finally:
+        os.environ = orig_environ
+
+
+def test_write_exclude_file_does_not_raise():
+    temporary_file = flexmock(
+        name='filename',
+        write=lambda mode: None,
+        flush=lambda: None,
+    )
+    flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
+
+    module._write_exclude_file(['exclude'])
+
+
+def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
+    module._write_exclude_file([])
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+
+
+def insert_platform_mock():
+    flexmock(module.platform).should_receive('node').and_return('host')
+
+
+def insert_datetime_mock():
+    flexmock(module).datetime = flexmock().should_receive('now').and_return(
+        flexmock().should_receive('isoformat').and_return('now').mock
+    ).mock
+
+
+CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
+
+
+def test_create_archive_should_call_borg_with_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(CREATE_COMMAND)
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
+    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes'))
+    insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': ['exclude'],
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        verbosity=VERBOSITY_SOME,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        verbosity=VERBOSITY_LOTS,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'compression': 'rle'},
+    )
+
+
+def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'one_file_system': True,
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'remote_path': 'borg1',
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'umask': 740},
+    )
+
+
+def test_create_archive_with_source_directories_glob_expands():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_non_matching_source_directories_glob_passes_through():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
+    flexmock(module).should_receive('_write_exclude_file')
+    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
diff --git a/borgmatic/tests/unit/borg/test_extract.py b/borgmatic/tests/unit/borg/test_extract.py
new file mode 100644
index 000000000..929c3243f
--- /dev/null
+++ b/borgmatic/tests/unit/borg/test_extract.py
@@ -0,0 +1,100 @@
+import sys
+
+from flexmock import flexmock
+
+from borgmatic.borg import extract as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+
+
+def insert_subprocess_never():
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').never()
+
+
+def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once()
+
+
+def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+    )
+
+
+def test_extract_last_archive_dry_run_without_any_archives_should_bail():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo'),
+        result='\n'.encode('utf-8'),
+    )
+    insert_subprocess_never()
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+    )
+
+
+def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--info'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--info'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=VERBOSITY_SOME,
+        repository='repo',
+    )
+
+
+def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--debug'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=VERBOSITY_LOTS,
+        repository='repo',
+    )
+
+
+def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        remote_path='borg1',
+    )
diff --git a/borgmatic/tests/unit/borg/test_prune.py b/borgmatic/tests/unit/borg/test_prune.py
new file mode 100644
index 000000000..379960add
--- /dev/null
+++ b/borgmatic/tests/unit/borg/test_prune.py
@@ -0,0 +1,93 @@
+from collections import OrderedDict
+
+from flexmock import flexmock
+
+from borgmatic.borg import prune as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+
+
+BASE_PRUNE_FLAGS = (
+    ('--keep-daily', '1'),
+    ('--keep-weekly', '2'),
+    ('--keep-monthly', '3'),
+)
+
+
+def test_make_prune_flags_should_return_flags_from_config():
+    retention_config = OrderedDict(
+        (
+            ('keep_daily', 1),
+            ('keep_weekly', 2),
+            ('keep_monthly', 3),
+        )
+    )
+
+    result = module._make_prune_flags(retention_config)
+
+    assert tuple(result) == BASE_PRUNE_FLAGS
+
+
+PRUNE_COMMAND = (
+    'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
+)
+
+
+def test_prune_archives_should_call_borg_with_parameters():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND)
+
+    module.prune_archives(
+        verbosity=None,
+        repository='repo',
+        retention_config=retention_config,
+    )
+
+
+def test_prune_archives_with_verbosity_some_should_call_borg_with_info_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--info', '--stats',))
+
+    module.prune_archives(
+        repository='repo',
+        verbosity=VERBOSITY_SOME,
+        retention_config=retention_config,
+    )
+
+
+def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats',))
+
+    module.prune_archives(
+        repository='repo',
+        verbosity=VERBOSITY_LOTS,
+        retention_config=retention_config,
+    )
+
+def test_prune_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1'))
+
+    module.prune_archives(
+        verbosity=None,
+        repository='repo',
+        retention_config=retention_config,
+        remote_path='borg1',
+    )
diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py
deleted file mode 100644
index 1cba04078..000000000
--- a/borgmatic/tests/unit/test_borg.py
+++ /dev/null
@@ -1,642 +0,0 @@
-from collections import OrderedDict
-from subprocess import STDOUT
-import sys
-import os
-
-from flexmock import flexmock
-import pytest
-
-from borgmatic import borg as module
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-def test_initialize_with_passphrase_should_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize({'encryption_passphrase': 'pass'}, command='borg')
-        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
-    finally:
-        os.environ = orig_environ
-
-
-def test_initialize_without_passphrase_should_not_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize({}, command='borg')
-        assert os.environ.get('BORG_PASSPHRASE') == None
-    finally:
-        os.environ = orig_environ
-
-def test_write_exclude_file_does_not_raise():
-    temporary_file = flexmock(
-        name='filename',
-        write=lambda mode: None,
-        flush=lambda: None,
-    )
-    flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
-
-    module._write_exclude_file(['exclude'])
-
-
-def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
-    module._write_exclude_file([])
-
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-    flexmock(module).subprocess = subprocess
-
-
-def insert_subprocess_never():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').never()
-    flexmock(module).subprocess = subprocess
-
-
-def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once()
-    flexmock(module).subprocess = subprocess
-
-
-def insert_platform_mock():
-    flexmock(module.platform).should_receive('node').and_return('host')
-
-
-def insert_datetime_mock():
-    flexmock(module).datetime = flexmock().should_receive('now').and_return(
-        flexmock().should_receive('isoformat').and_return('now').mock
-    ).mock
-
-
-CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
-
-
-def test_create_archive_should_call_borg_with_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(CREATE_COMMAND)
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-        command='borg',
-    )
-
-
-def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
-    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes'))
-    insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes'))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': ['exclude'],
-        },
-        storage_config={},
-        command='borg',
-    )
-
-
-def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        verbosity=VERBOSITY_SOME,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-        command='borg',
-    )
-
-
-def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        verbosity=VERBOSITY_LOTS,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-        command='borg',
-    )
-
-
-def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={'compression': 'rle'},
-        command='borg',
-    )
-
-
-def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'one_file_system': True,
-            'exclude_patterns': None,
-        },
-        storage_config={},
-        command='borg',
-    )
-
-
-def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'remote_path': 'borg1',
-            'exclude_patterns': None,
-        },
-        storage_config={},
-        command='borg',
-    )
-
-
-def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={'umask': 740},
-        command='borg',
-    )
-
-
-def test_create_archive_with_source_directories_glob_expands():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
-    insert_platform_mock()
-    insert_datetime_mock()
-    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo*'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-        command='borg',
-    )
-
-
-def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*'))
-    insert_platform_mock()
-    insert_datetime_mock()
-    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo*'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-        command='borg',
-    )
-
-
-def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
-    flexmock(module).should_receive('_write_exclude_file')
-    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
-    insert_platform_mock()
-    insert_datetime_mock()
-    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo*'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-        command='borg',
-    )
-
-
-BASE_PRUNE_FLAGS = (
-    ('--keep-daily', '1'),
-    ('--keep-weekly', '2'),
-    ('--keep-monthly', '3'),
-)
-
-
-def test_make_prune_flags_should_return_flags_from_config():
-    retention_config = OrderedDict(
-        (
-            ('keep_daily', 1),
-            ('keep_weekly', 2),
-            ('keep_monthly', 3),
-        )
-    )
-
-    result = module._make_prune_flags(retention_config)
-
-    assert tuple(result) == BASE_PRUNE_FLAGS
-
-
-PRUNE_COMMAND = (
-    'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
-)
-
-
-def test_prune_archives_should_call_borg_with_parameters():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND)
-
-    module.prune_archives(
-        verbosity=None,
-        repository='repo',
-        retention_config=retention_config,
-        command='borg',
-    )
-
-
-def test_prune_archives_with_verbosity_some_should_call_borg_with_info_parameter():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--info', '--stats',))
-
-    module.prune_archives(
-        repository='repo',
-        verbosity=VERBOSITY_SOME,
-        retention_config=retention_config,
-        command='borg',
-    )
-
-
-def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats',))
-
-    module.prune_archives(
-        repository='repo',
-        verbosity=VERBOSITY_LOTS,
-        retention_config=retention_config,
-        command='borg',
-    )
-
-def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1'))
-
-    module.prune_archives(
-        verbosity=None,
-        repository='repo',
-        retention_config=retention_config,
-        command='borg',
-        remote_path='borg1',
-    )
-
-
-def test_parse_checks_returns_them_as_tuple():
-    checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']})
-
-    assert checks == ('foo', 'bar')
-
-
-def test_parse_checks_with_missing_value_returns_defaults():
-    checks = module._parse_checks({})
-
-    assert checks == module.DEFAULT_CHECKS
-
-
-def test_parse_checks_with_blank_value_returns_defaults():
-    checks = module._parse_checks({'checks': []})
-
-    assert checks == module.DEFAULT_CHECKS
-
-
-def test_parse_checks_with_disabled_returns_no_checks():
-    checks = module._parse_checks({'checks': ['disabled']})
-
-    assert checks == ()
-
-
-def test_make_check_flags_with_checks_returns_flags():
-    flags = module._make_check_flags(('repository',))
-
-    assert flags == ('--repository-only',)
-
-
-def test_make_check_flags_with_extract_check_does_not_make_extract_flag():
-    flags = module._make_check_flags(('extract',))
-
-    assert flags == ()
-
-
-def test_make_check_flags_with_default_checks_returns_no_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS)
-
-    assert flags == ()
-
-
-def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
-    flags = module._make_check_flags(('repository',), check_last=3)
-
-    assert flags == ('--repository-only', '--last', '3')
-
-
-def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
-
-    assert flags == ('--last', '3')
-
-
-@pytest.mark.parametrize(
-    'checks',
-    (
-        ('repository',),
-        ('archives',),
-        ('repository', 'archives'),
-        ('repository', 'archives', 'other'),
-    ),
-)
-def test_check_archives_should_call_borg_with_parameters(checks):
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo'),
-        stdout=stdout, stderr=STDOUT,
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='borg',
-    )
-
-
-def test_check_archives_with_extract_check_should_call_extract_only():
-    checks = ('extract',)
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').never()
-    flexmock(module).should_receive('extract_last_archive_dry_run').once()
-    insert_subprocess_never()
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='borg',
-    )
-
-
-def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
-    checks = ('repository',)
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--info'),
-        stdout=None, stderr=STDOUT,
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.check_archives(
-        verbosity=VERBOSITY_SOME,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='borg',
-    )
-
-
-def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    checks = ('repository',)
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--debug'),
-        stdout=None, stderr=STDOUT,
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.check_archives(
-        verbosity=VERBOSITY_LOTS,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='borg',
-    )
-
-
-def test_check_archives_without_any_checks_should_bail():
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(())
-    insert_subprocess_never()
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='borg',
-    )
-
-
-def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
-    checks = ('repository',)
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--remote-path', 'borg1'),
-        stdout=stdout, stderr=STDOUT,
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='borg',
-        remote_path='borg1',
-    )
-
-
-def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo'),
-        result='archive1\narchive2\n'.encode('utf-8'),
-    )
-    insert_subprocess_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2'),
-    )
-
-    module.extract_last_archive_dry_run(
-        verbosity=None,
-        repository='repo',
-        command='borg',
-    )
-
-
-def test_extract_last_archive_dry_run_without_any_archives_should_bail():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo'),
-        result='\n'.encode('utf-8'),
-    )
-    insert_subprocess_never()
-
-    module.extract_last_archive_dry_run(
-        verbosity=None,
-        repository='repo',
-        command='borg',
-    )
-
-
-def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--info'),
-        result='archive1\narchive2\n'.encode('utf-8'),
-    )
-    insert_subprocess_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2', '--info'),
-    )
-
-    module.extract_last_archive_dry_run(
-        verbosity=VERBOSITY_SOME,
-        repository='repo',
-        command='borg',
-    )
-
-
-def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--debug'),
-        result='archive1\narchive2\n'.encode('utf-8'),
-    )
-    insert_subprocess_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'),
-    )
-
-    module.extract_last_archive_dry_run(
-        verbosity=VERBOSITY_LOTS,
-        repository='repo',
-        command='borg',
-    )
-
-
-def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
-        result='archive1\narchive2\n'.encode('utf-8'),
-    )
-    insert_subprocess_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'),
-    )
-
-    module.extract_last_archive_dry_run(
-        verbosity=None,
-        repository='repo',
-        command='borg',
-        remote_path='borg1',
-    )

From 674a6153f38f999c2d0388021268649fb70cae39 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 5 Aug 2017 22:26:28 -0700
Subject: [PATCH 170/189] Fix imports of borg/*.py modules now that they've
 been split out.

---
 borgmatic/commands/borgmatic.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index c5c7a28d7..5085b65b1 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -4,7 +4,7 @@ import os
 from subprocess import CalledProcessError
 import sys
 
-from borgmatic import borg
+from borgmatic.borg import check, create, prune
 from borgmatic.config import collect, convert, validate
 
 
@@ -92,20 +92,20 @@ def main():  # pragma: no cover
             )
             remote_path = location.get('remote_path')
 
-            borg.initialize(storage)
+            create.initialize(storage)
 
             for repository in location['repositories']:
                 if args.prune:
-                    borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+                    prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
                 if args.create:
-                    borg.create_archive(
+                    create.create_archive(
                         args.verbosity,
                         repository,
                         location,
                         storage,
                     )
                 if args.check:
-                    borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
+                    check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

From 51095cd4192da82106cda54338c4242bd1bd89db Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 5 Aug 2017 22:26:38 -0700
Subject: [PATCH 171/189] Remove unused imports.

---
 borgmatic/commands/convert_config.py  | 1 -
 borgmatic/commands/generate_config.py | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py
index ca001e0ca..6a269c8e8 100644
--- a/borgmatic/commands/convert_config.py
+++ b/borgmatic/commands/convert_config.py
@@ -7,7 +7,6 @@ import textwrap
 
 from ruamel import yaml
 
-from borgmatic import borg
 from borgmatic.config import convert, generate, legacy, validate
 
 
diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py
index ba88ddf8c..92aa797ea 100644
--- a/borgmatic/commands/generate_config.py
+++ b/borgmatic/commands/generate_config.py
@@ -4,9 +4,6 @@ import os
 from subprocess import CalledProcessError
 import sys
 
-from ruamel import yaml
-
-from borgmatic import borg
 from borgmatic.config import convert, generate, validate
 
 

From 10cac46f4c56f47b61e1b938a4f8443f39c369d6 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 5 Aug 2017 23:32:39 -0700
Subject: [PATCH 172/189] #12, #35: Support for Borg --exclude-from,
 --exclude-caches, and --exclude-if-present options.

---
 NEWS                                     |  4 +
 README.md                                |  2 +-
 borgmatic/borg/create.py                 | 32 +++++++-
 borgmatic/config/schema.yaml             | 18 +++++
 borgmatic/tests/unit/borg/test_create.py | 96 +++++++++++++++++++++---
 setup.py                                 |  2 +-
 6 files changed, 136 insertions(+), 18 deletions(-)

diff --git a/NEWS b/NEWS
index 6222fea6c..66447b2c3 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+1.1.6
+
+ * #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.
+
 1.1.5
 
  * #34: New "extract" consistency check that performs a dry-run extraction of the most recent
diff --git a/README.md b/README.md
index 057388bf7..6840d06e9 100644
--- a/README.md
+++ b/README.md
@@ -121,7 +121,7 @@ However, see below about special cases.
 borgmatic changed its configuration file format in version 1.1.0 from
 INI-style to YAML. This better supports validation, and has a more natural way
 to express lists of values. To upgrade your existing configuration, first
-upgrade to the new version of borgmatic:
+upgrade to the new version of borgmatic.
 
 As of version 1.1.0, borgmatic no longer supports Python 2. If you were
 already running borgmatic with Python 3, then you can simply upgrade borgmatic
diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py
index 21af701c8..2696e7515 100644
--- a/borgmatic/borg/create.py
+++ b/borgmatic/borg/create.py
@@ -31,12 +31,33 @@ def _write_exclude_file(exclude_patterns=None):
     return exclude_file
 
 
+def _make_exclude_flags(location_config, exclude_patterns_filename=None):
+    '''
+    Given a location config dict with various exclude options, and a filename containing any exclude
+    patterns, return the corresponding Borg flags as a tuple.
+    '''
+    exclude_filenames = tuple(location_config.get('exclude_from', ())) + (
+        (exclude_patterns_filename,) if exclude_patterns_filename else ()
+    )
+    exclude_from_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-from', exclude_filename)
+            for exclude_filename in exclude_filenames
+        )
+    )
+    caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
+    if_present = location_config.get('exclude_if_present')
+    if_present_flags = ('--exclude-if-present', if_present) if if_present else ()
+
+    return exclude_from_flags + caches_flag + if_present_flags
+
+
 def create_archive(
     verbosity, repository, location_config, storage_config,
 ):
     '''
-    Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
-    repository path, a list of exclude patterns, create a Borg archive.
+    Given a vebosity flag, a local or remote repository path, a location config dict, and a storage
+    config dict, create a Borg archive.
     '''
     sources = tuple(
         itertools.chain.from_iterable(
@@ -45,8 +66,11 @@ def create_archive(
         )
     )
 
-    exclude_file = _write_exclude_file(location_config.get('exclude_patterns'))
-    exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
+    exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns'))
+    exclude_flags = _make_exclude_flags(
+        location_config,
+        exclude_patterns_file.name if exclude_patterns_file else None,
+    )
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
     umask = storage_config.get('umask', None)
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index 567ec7329..f7e4c4f0f 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -45,6 +45,24 @@ map:
                     - '*.pyc'
                     - /home/*/.cache
                     - /etc/ssl
+            exclude_from:
+                seq:
+                    - type: scalar
+                desc: |
+                    Read exclude patterns from one or more separate named files, one pattern per
+                    line.
+                example:
+                    - /etc/borgmatic/excludes
+            exclude_caches:
+                type: bool
+                desc: |
+                    Exclude directories that contain a CACHEDIR.TAG file. See
+                    http://www.brynosaurus.com/cachedir/spec.html for details.
+                example: true
+            exclude_if_present:
+                type: scalar
+                desc: Exclude directories that contain a file with the given filename.
+                example: .nobackup
     storage:
         desc: |
             Repository storage options. See
diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py
index 3b4d2689c..6d2aa4589 100644
--- a/borgmatic/tests/unit/borg/test_create.py
+++ b/borgmatic/tests/unit/borg/test_create.py
@@ -58,11 +58,72 @@ def insert_datetime_mock():
     ).mock
 
 
+def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_patterns': ['*.pyc', '/var']},
+        exclude_patterns_filename='/tmp/excludes',
+    )
+
+    assert exclude_flags == ('--exclude-from', '/tmp/excludes')
+
+
+def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_from': ['excludes', 'other']},
+    )
+
+    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', 'other')
+
+
+def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config():
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_from': ['excludes']},
+        exclude_patterns_filename='/tmp/excludes',
+    )
+
+    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
+
+
+def test_make_exclude_flags_includes_exclude_caches_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_caches': True},
+    )
+
+    assert exclude_flags == ('--exclude-caches',)
+
+
+def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_caches': False},
+    )
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_if_present': 'exclude_me'},
+    )
+
+    assert exclude_flags == ('--exclude-if-present', 'exclude_me')
+
+
+def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
+    exclude_flags = module._make_exclude_flags(location_config={})
+
+    assert exclude_flags == ()
+
+
 CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
 
 
 def test_create_archive_should_call_borg_with_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND)
     insert_platform_mock()
     insert_datetime_mock()
@@ -80,8 +141,10 @@ def test_create_archive_should_call_borg_with_parameters():
 
 
 def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
-    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes'))
-    insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes'))
+    exclude_flags = ('--exclude-from', 'excludes')
+    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes'))
+    flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
+    insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
     insert_platform_mock()
     insert_datetime_mock()
 
@@ -98,7 +161,8 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
 
 
 def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
     insert_platform_mock()
     insert_datetime_mock()
@@ -116,7 +180,8 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter
 
 
 def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -134,7 +199,8 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete
 
 
 def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -152,7 +218,8 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param
 
 
 def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
     insert_platform_mock()
     insert_datetime_mock()
@@ -171,7 +238,8 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
 
 
 def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -190,7 +258,8 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
 
 
 def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -208,7 +277,8 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
 
 
 def test_create_archive_with_source_directories_glob_expands():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -227,7 +297,8 @@ def test_create_archive_with_source_directories_glob_expands():
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*'))
     insert_platform_mock()
     insert_datetime_mock()
@@ -246,7 +317,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 
 def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
-    flexmock(module).should_receive('_write_exclude_file')
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
diff --git a/setup.py b/setup.py
index 2ff96905a..a842ab544 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.5'
+VERSION = '1.1.6'
 
 
 setup(

From 3664ac741899c55d3dfb20a86f9da8b1046785db Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 5 Aug 2017 23:33:08 -0700
Subject: [PATCH 173/189] Added tag 1.1.6 for changeset 4daa944c122c

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 50ee67dcf..58595242f 100644
--- a/.hgtags
+++ b/.hgtags
@@ -36,3 +36,4 @@ f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2
 3f838f661546e04529b453aa443529b432afc243 1.1.3
 3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4
 64ca13bfe050f656b44ed2eb1c3db045bfddd133 1.1.5
+4daa944c122c572b9b56bfcac3f4e2869181c630 1.1.6

From 37ae34a432483d1af1a33032113c568954b588c3 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 26 Aug 2017 16:07:30 -0700
Subject: [PATCH 174/189] When pruning, make highest verbosity level list
 archives kept and pruned.

---
 NEWS                                    | 4 ++++
 borgmatic/borg/prune.py                 | 2 +-
 borgmatic/tests/unit/borg/test_prune.py | 2 +-
 setup.py                                | 2 +-
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index 66447b2c3..677aec68a 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+1.1.7.dev0
+
+ * When pruning, make highest verbosity level list archives kept and pruned.
+
 1.1.6
 
  * #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.
diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py
index 8f52cb4c0..7f0c03815 100644
--- a/borgmatic/borg/prune.py
+++ b/borgmatic/borg/prune.py
@@ -33,7 +33,7 @@ def prune_archives(verbosity, repository, retention_config, remote_path=None):
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--stats'),
+        VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
     }.get(verbosity, ())
 
     full_command = (
diff --git a/borgmatic/tests/unit/borg/test_prune.py b/borgmatic/tests/unit/borg/test_prune.py
index 379960add..ffcf32ad7 100644
--- a/borgmatic/tests/unit/borg/test_prune.py
+++ b/borgmatic/tests/unit/borg/test_prune.py
@@ -70,7 +70,7 @@ def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_paramete
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats',))
+    insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats', '--list'))
 
     module.prune_archives(
         repository='repo',
diff --git a/setup.py b/setup.py
index a842ab544..f70e334ae 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.6'
+VERSION = '1.1.7.dev0'
 
 
 setup(

From 50c4f6f2a104ed72e26e3472a087623d9812596e Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 26 Aug 2017 16:13:41 -0700
Subject: [PATCH 175/189] Adding documentation note about pruning happening
 before archiving.

---
 README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.md b/README.md
index 6840d06e9..588c07ca5 100644
--- a/README.md
+++ b/README.md
@@ -183,6 +183,11 @@ If you'd like to see the available command-line arguments, view the help:
 
     borgmatic --help
 
+Note that borgmatic prunes archives *before* creating an archive, so as to
+free up space for archiving. This means that when a borgmatic run finishes,
+there may still be prune-able archives. Not to worry, as they will get cleaned
+up at the start of the next run.
+
 ### Verbosity
 
 By default, the backup will proceed silently except in the case of errors. But

From d127e73590e85f3c93b678b36e22736d08f01010 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 26 Aug 2017 16:18:53 -0700
Subject: [PATCH 176/189] Clarification of Python 3 pip usage in documentation.

---
 NEWS      |  1 +
 README.md | 19 ++++++++-----------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/NEWS b/NEWS
index 677aec68a..2df29ece1 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,7 @@
 1.1.7.dev0
 
  * When pruning, make highest verbosity level list archives kept and pruned.
+ * Clarification of Python 3 pip usage in documentation.
 
 1.1.6
 
diff --git a/README.md b/README.md
index 588c07ca5..1e10d4455 100644
--- a/README.md
+++ b/README.md
@@ -61,10 +61,10 @@ key-based ssh access to the desired user account on the remote host.
 
 To install borgmatic, run the following command to download and install it:
 
-    sudo pip install --upgrade borgmatic
+    sudo pip3 install --upgrade borgmatic
 
-Make sure you're using Python 3, as borgmatic does not support Python 2. (You
-may have to use "pip3" or similar instead of "pip".)
+Note that your pip binary may have a different name than "pip3". Make sure
+you're using Python 3, as borgmatic does not support Python 2.
 
 ## Configuration
 
@@ -108,10 +108,7 @@ default, the traditional /etc/borgmatic/config.yaml as well.
 In general, all you should need to do to upgrade borgmatic is run the
 following:
 
-    sudo pip install --upgrade borgmatic
-
-(You may have to use "pip3" or similar instead of "pip", so Python 3 gets
-used.)
+    sudo pip3 install --upgrade borgmatic
 
 However, see below about special cases.
 
@@ -127,7 +124,7 @@ As of version 1.1.0, borgmatic no longer supports Python 2. If you were
 already running borgmatic with Python 3, then you can simply upgrade borgmatic
 in-place:
 
-    sudo pip install --upgrade borgmatic
+    sudo pip3 install --upgrade borgmatic
 
 But if you were running borgmatic with Python 2, uninstall and reinstall instead:
 
@@ -162,8 +159,8 @@ your borgmatic configuration files.
 If you were already using Borg with atticmatic, then you can easily upgrade
 from atticmatic to borgmatic. Simply run the following commands:
 
-    sudo pip uninstall atticmatic
-    sudo pip install borgmatic
+    sudo pip3 uninstall atticmatic
+    sudo pip3 install borgmatic
 
 That's it! borgmatic will continue using your /etc/borgmatic configuration
 files.
@@ -252,7 +249,7 @@ borgmatic to run.
 
 First install tox, which is used for setting up testing environments:
 
-    pip install tox
+    pip3 install tox
 
 Then, to actually run tests, run:
 

From 3af92f8b9286d98f9c83c166772573216e13765c Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 27 Aug 2017 10:01:49 -0700
Subject: [PATCH 177/189] Fix for traceback when "exclude_from" value is empty
 in configuration file.

---
 NEWS                                     |  1 +
 borgmatic/borg/create.py                 |  2 +-
 borgmatic/tests/unit/borg/test_create.py | 10 ++++++++++
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 2df29ece1..2e5abe7e4 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
 1.1.7.dev0
 
+ * Fix for traceback when "exclude_from" value is empty in configuration file.
  * When pruning, make highest verbosity level list archives kept and pruned.
  * Clarification of Python 3 pip usage in documentation.
 
diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py
index 2696e7515..5073cc57f 100644
--- a/borgmatic/borg/create.py
+++ b/borgmatic/borg/create.py
@@ -36,7 +36,7 @@ def _make_exclude_flags(location_config, exclude_patterns_filename=None):
     Given a location config dict with various exclude options, and a filename containing any exclude
     patterns, return the corresponding Borg flags as a tuple.
     '''
-    exclude_filenames = tuple(location_config.get('exclude_from', ())) + (
+    exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
         (exclude_patterns_filename,) if exclude_patterns_filename else ()
     )
     exclude_from_flags = tuple(
diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py
index 6d2aa4589..6f9b5a380 100644
--- a/borgmatic/tests/unit/borg/test_create.py
+++ b/borgmatic/tests/unit/borg/test_create.py
@@ -88,6 +88,16 @@ def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_excl
     assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
 
 
+def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty():
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_from': None},
+    )
+
+    assert exclude_flags == ()
+
+
 def test_make_exclude_flags_includes_exclude_caches_when_true_in_config():
     exclude_flags = module._make_exclude_flags(
         location_config={'exclude_caches': True},

From 95533d2b3103f607e541c8b5ef4d80569fd3225b Mon Sep 17 00:00:00 2001
From: Michele Lazzeri <19763535+mlazze@users.noreply.github.com>
Date: Sun, 3 Sep 2017 20:13:14 +0200
Subject: [PATCH 178/189] Added storage.archive_name_format to config (#16)

* Added storage.archive_name_format to config
---
 borgmatic/borg/create.py                 | 11 ++--
 borgmatic/config/schema.yaml             | 12 +++-
 borgmatic/tests/unit/borg/test_create.py | 79 +++++++++++++-----------
 borgmatic/tests/unit/borg/test_prune.py  | 19 ++++++
 4 files changed, 78 insertions(+), 43 deletions(-)

diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py
index 5073cc57f..c2f2cf34a 100644
--- a/borgmatic/borg/create.py
+++ b/borgmatic/borg/create.py
@@ -1,8 +1,6 @@
-from datetime import datetime
 import glob
 import itertools
 import os
-import platform
 import subprocess
 import tempfile
 
@@ -82,15 +80,16 @@ def create_archive(
         VERBOSITY_SOME: ('--info', '--stats',),
         VERBOSITY_LOTS: ('--debug', '--list', '--stats'),
     }.get(verbosity, ())
+    default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+    archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
 
     full_command = (
         'borg', 'create',
-        '{repository}::{hostname}-{timestamp}'.format(
+        '{repository}::{archive_name_format}'.format(
             repository=repository,
-            hostname=platform.node(),
-            timestamp=datetime.now().isoformat(),
+            archive_name_format=archive_name_format,
         ),
     ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
         remote_path_flags + umask_flags + verbosity_flags
 
-    subprocess.check_call(full_command)
+    subprocess.check_call(full_command)
\ No newline at end of file
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index f7e4c4f0f..74d99fe83 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -88,6 +88,13 @@ map:
                 type: scalar
                 desc: Umask to be used for borg create.
                 example: 0077
+            archive_name_format:
+                type: scalar
+                desc: |
+                    Name of the archive. Borg placeholders can be used. See
+                    https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-placeholders
+                    Default is "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}"
+                example: "{hostname}-documents-{now}"
     retention:
         desc: |
             Retention policy for how many backups to keep in each category. See
@@ -119,7 +126,10 @@ map:
                 example: 1
             prefix:
                 type: scalar
-                desc: When pruning, only consider archive names starting with this prefix.
+                desc: |
+                    When pruning, only consider archive names starting with this prefix.
+                    Borg placeholders can be used. See
+                    https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-placeholders
                 example: sourcehostname
     consistency:
         desc: |
diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py
index 6f9b5a380..743c0946d 100644
--- a/borgmatic/tests/unit/borg/test_create.py
+++ b/borgmatic/tests/unit/borg/test_create.py
@@ -48,16 +48,6 @@ def insert_subprocess_mock(check_call_command, **kwargs):
     subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
 
 
-def insert_platform_mock():
-    flexmock(module.platform).should_receive('node').and_return('host')
-
-
-def insert_datetime_mock():
-    flexmock(module).datetime = flexmock().should_receive('now').and_return(
-        flexmock().should_receive('isoformat').and_return('now').mock
-    ).mock
-
-
 def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
     exclude_flags = module._make_exclude_flags(
         location_config={'exclude_patterns': ['*.pyc', '/var']},
@@ -128,15 +118,14 @@ def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
     assert exclude_flags == ()
 
 
-CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
+DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 
 
 def test_create_archive_should_call_borg_with_parameters():
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND)
-    insert_platform_mock()
-    insert_datetime_mock()
 
     module.create_archive(
         verbosity=None,
@@ -155,8 +144,6 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
     flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes'))
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
     insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
-    insert_platform_mock()
-    insert_datetime_mock()
 
     module.create_archive(
         verbosity=None,
@@ -174,8 +161,6 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
-    insert_platform_mock()
-    insert_datetime_mock()
 
     module.create_archive(
         verbosity=VERBOSITY_SOME,
@@ -193,8 +178,6 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
-    insert_platform_mock()
-    insert_datetime_mock()
 
     module.create_archive(
         verbosity=VERBOSITY_LOTS,
@@ -212,8 +195,6 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
-    insert_platform_mock()
-    insert_datetime_mock()
 
     module.create_archive(
         verbosity=None,
@@ -231,8 +212,6 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
-    insert_platform_mock()
-    insert_datetime_mock()
 
     module.create_archive(
         verbosity=None,
@@ -251,8 +230,6 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
-    insert_platform_mock()
-    insert_datetime_mock()
 
     module.create_archive(
         verbosity=None,
@@ -271,8 +248,6 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
-    insert_platform_mock()
-    insert_datetime_mock()
 
     module.create_archive(
         verbosity=None,
@@ -289,9 +264,7 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
-    insert_platform_mock()
-    insert_datetime_mock()
+    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
@@ -309,9 +282,7 @@ def test_create_archive_with_source_directories_glob_expands():
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*'))
-    insert_platform_mock()
-    insert_datetime_mock()
+    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
     module.create_archive(
@@ -329,9 +300,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
-    insert_platform_mock()
-    insert_datetime_mock()
+    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
@@ -344,3 +313,41 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
         },
         storage_config={},
     )
+
+
+def test_create_archive_with_archive_name_format_without_placeholders():
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={
+            'archive_name_format': 'ARCHIVE_NAME',
+        },
+    )
+
+
+def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
+    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={
+            'archive_name_format': 'Documents_{hostname}-{now}',
+        },
+    )
diff --git a/borgmatic/tests/unit/borg/test_prune.py b/borgmatic/tests/unit/borg/test_prune.py
index ffcf32ad7..003b2cbb6 100644
--- a/borgmatic/tests/unit/borg/test_prune.py
+++ b/borgmatic/tests/unit/borg/test_prune.py
@@ -32,6 +32,24 @@ def test_make_prune_flags_should_return_flags_from_config():
     assert tuple(result) == BASE_PRUNE_FLAGS
 
 
+def test_make_prune_flags_accepts_prefix_with_placeholders():
+    retention_config = OrderedDict(
+        (
+            ('keep_daily', 1),
+            ('prefix', 'Documents_{hostname}-{now}'),
+        )
+    )
+
+    result = module._make_prune_flags(retention_config)
+
+    expected = (
+        ('--keep-daily', '1'),
+        ('--prefix', 'Documents_{hostname}-{now}'),
+    )
+
+    assert tuple(result) == expected
+
+
 PRUNE_COMMAND = (
     'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
 )
@@ -78,6 +96,7 @@ def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_paramete
         retention_config=retention_config,
     )
 
+
 def test_prune_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
     retention_config = flexmock()
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(

From bb18a9a3f20adbf89b4c9a92d8ff5ee5ebadd121 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 3 Sep 2017 11:33:07 -0700
Subject: [PATCH 179/189] Update NEWS and AUTHORS for release.

---
 AUTHORS  | 2 ++
 NEWS     | 3 ++-
 setup.py | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 95d2df819..bd99af8ca 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -2,4 +2,6 @@ Dan Helfman <witten@torsion.org>: Main developer
 
 Alexander Görtz: Python 3 compatibility
 Henning Schroeder: Copy editing
+Michele Lazzeri: Custom archive names
 Robin `ypid` Schneider: Support additional options of Borg
+Scott Squires: Custom archive names
diff --git a/NEWS b/NEWS
index 2e5abe7e4..04a3d5a77 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
-1.1.7.dev0
+1.1.7
 
+ * #28: Add "archive_name_format" to configuration for customizing archive names.
  * Fix for traceback when "exclude_from" value is empty in configuration file.
  * When pruning, make highest verbosity level list archives kept and pruned.
  * Clarification of Python 3 pip usage in documentation.
diff --git a/setup.py b/setup.py
index f70e334ae..bd939266c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.7.dev0'
+VERSION = '1.1.7'
 
 
 setup(

From 6c4f641c1ef42a44bba0e3b34531d9578c723cd1 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sun, 3 Sep 2017 11:33:10 -0700
Subject: [PATCH 180/189] Added tag 1.1.7 for changeset ec7949a14a20

---
 .hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags b/.hgtags
index 58595242f..23c952873 100644
--- a/.hgtags
+++ b/.hgtags
@@ -37,3 +37,4 @@ f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2
 3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4
 64ca13bfe050f656b44ed2eb1c3db045bfddd133 1.1.5
 4daa944c122c572b9b56bfcac3f4e2869181c630 1.1.6
+ec7949a14a2051616f7cdcb8e05555f02e024ae8 1.1.7

From d30caa422e693ef1c7796899198ba9dd4f5a28d0 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Fri, 8 Sep 2017 21:25:42 -0700
Subject: [PATCH 181/189] #39: Fix to make /etc/borgmatic/config.yaml optional
 rather than required when using the default config paths.

---
 NEWS                                              |  4 ++++
 borgmatic/commands/borgmatic.py                   |  6 ++----
 borgmatic/config/collect.py                       |  5 ++++-
 .../tests/integration/commands/test_borgmatic.py  |  4 ++--
 borgmatic/tests/unit/config/test_collect.py       | 15 ++++++++++++++-
 setup.py                                          |  2 +-
 6 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/NEWS b/NEWS
index 04a3d5a77..6bd9ece3f 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+1.1.8
+ * #39: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default
+   config paths.
+
 1.1.7
 
  * #28: Add "archive_name_format" to configuration for customizing archive names.
diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 5085b65b1..2fdff34ad 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -9,8 +9,6 @@ from borgmatic.config import collect, convert, validate
 
 
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
-DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d']
-DEFAULT_EXCLUDES_PATH = '/etc/borgmatic/excludes'
 
 
 def parse_arguments(*arguments):
@@ -30,8 +28,8 @@ def parse_arguments(*arguments):
         '-c', '--config',
         nargs='+',
         dest='config_paths',
-        default=DEFAULT_CONFIG_PATHS,
-        help='Configuration filenames or directories, defaults to: {}'.format(' '.join(DEFAULT_CONFIG_PATHS)),
+        default=collect.DEFAULT_CONFIG_PATHS,
+        help='Configuration filenames or directories, defaults to: {}'.format(' '.join(collect.DEFAULT_CONFIG_PATHS)),
     )
     parser.add_argument(
         '--excludes',
diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py
index dbe6f2fe7..4b8b88e18 100644
--- a/borgmatic/config/collect.py
+++ b/borgmatic/config/collect.py
@@ -1,6 +1,9 @@
 import os
 
 
+DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d']
+
+
 def collect_config_filenames(config_paths):
     '''
     Given a sequence of config paths, both filenames and directories, resolve that to just an
@@ -14,7 +17,7 @@ def collect_config_filenames(config_paths):
     for path in config_paths:
         exists = os.path.exists(path)
 
-        if os.path.realpath(path) == '/etc/borgmatic.d' and not exists:
+        if os.path.realpath(path) in DEFAULT_CONFIG_PATHS and not exists:
             continue
 
         if not os.path.isdir(path) or not exists:
diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py
index 2b82ecfb1..1d6aa5c97 100644
--- a/borgmatic/tests/integration/commands/test_borgmatic.py
+++ b/borgmatic/tests/integration/commands/test_borgmatic.py
@@ -9,7 +9,7 @@ from borgmatic.commands import borgmatic as module
 def test_parse_arguments_with_no_arguments_uses_defaults():
     parser = module.parse_arguments()
 
-    assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
+    assert parser.config_paths == module.collect.DEFAULT_CONFIG_PATHS
     assert parser.excludes_filename == None
     assert parser.verbosity is None
 
@@ -32,7 +32,7 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list():
 def test_parse_arguments_with_verbosity_flag_overrides_default():
     parser = module.parse_arguments('--verbosity', '1')
 
-    assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
+    assert parser.config_paths == module.collect.DEFAULT_CONFIG_PATHS
     assert parser.excludes_filename == None
     assert parser.verbosity == 1
 
diff --git a/borgmatic/tests/unit/config/test_collect.py b/borgmatic/tests/unit/config/test_collect.py
index 2adee63e0..c2572c8c5 100644
--- a/borgmatic/tests/unit/config/test_collect.py
+++ b/borgmatic/tests/unit/config/test_collect.py
@@ -32,6 +32,19 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
     )
 
 
+def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist():
+    config_paths = ('config.yaml', '/etc/borgmatic/config.yaml')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
+    mock_path.should_receive('exists').with_args('/etc/borgmatic/config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic/config.yaml').and_return(True)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == ('config.yaml',)
+
+
 def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist():
     config_paths = ('config.yaml', '/etc/borgmatic.d')
     mock_path = flexmock(module.os.path)
@@ -45,7 +58,7 @@ def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist
     assert config_filenames == ('config.yaml',)
 
 
-def test_collect_config_filenames_includes_directory_if_it_does_not_exist():
+def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist():
     config_paths = ('config.yaml', '/my/directory')
     mock_path = flexmock(module.os.path)
     mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
diff --git a/setup.py b/setup.py
index bd939266c..d3b8307e6 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.7'
+VERSION = '1.1.8'
 
 
 setup(

From f3d6d7c0a3674c30e4122f6cac590f3900170682 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 9 Sep 2017 17:23:31 -0700
Subject: [PATCH 182/189] #29: Support for using tilde in source directory path
 to reference home directory.

---
 NEWS                                     |  3 ++
 borgmatic/borg/create.py                 | 14 ++++++-
 borgmatic/config/schema.yaml             |  3 +-
 borgmatic/tests/unit/borg/test_create.py | 52 +++++++++++++++++++-----
 setup.py                                 |  2 +-
 5 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/NEWS b/NEWS
index 6bd9ece3f..102967909 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,6 @@
+1.1.9.dev0
+ * #29: Support for using tilde in source directory path to reference home directory.
+
 1.1.8
  * #39: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default
    config paths.
diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py
index c2f2cf34a..47315dd10 100644
--- a/borgmatic/borg/create.py
+++ b/borgmatic/borg/create.py
@@ -14,6 +14,16 @@ def initialize(storage_config):
         os.environ['BORG_PASSPHRASE'] = passphrase
 
 
+def _expand_directory(directory):
+    '''
+    Given a directory path, expand any tilde (representing a user's home directory) and any globs
+    therein. Return a list of one or more resulting paths.
+    '''
+    expanded_directory = os.path.expanduser(directory)
+
+    return glob.glob(expanded_directory) or [expanded_directory]
+
+
 def _write_exclude_file(exclude_patterns=None):
     '''
     Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
@@ -59,7 +69,7 @@ def create_archive(
     '''
     sources = tuple(
         itertools.chain.from_iterable(
-            glob.glob(directory) or [directory]
+            _expand_directory(directory)
             for directory in location_config['source_directories']
         )
     )
@@ -92,4 +102,4 @@ def create_archive(
     ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
         remote_path_flags + umask_flags + verbosity_flags
 
-    subprocess.check_call(full_command)
\ No newline at end of file
+    subprocess.check_call(full_command)
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index 74d99fe83..d559ee3f3 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -12,7 +12,8 @@ map:
                 required: true
                 seq:
                     - type: scalar
-                desc: List of source directories to backup (required). Globs are expanded.
+                desc: |
+                    List of source directories to backup (required). Globs and tildes are expanded.
                 example:
                     - /home
                     - /etc
diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py
index 743c0946d..ebbee972f 100644
--- a/borgmatic/tests/unit/borg/test_create.py
+++ b/borgmatic/tests/unit/borg/test_create.py
@@ -28,6 +28,24 @@ def test_initialize_without_passphrase_should_not_set_environment():
         os.environ = orig_environ
 
 
+def test_expand_directory_with_basic_path_passes_it_through():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo')
+    flexmock(module.glob).should_receive('glob').and_return([])
+
+    paths = module._expand_directory('foo')
+
+    assert paths == ['foo']
+
+
+def test_expand_directory_with_glob_expands():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
+    flexmock(module.glob).should_receive('glob').and_return(['foo', 'food'])
+
+    paths = module._expand_directory('foo*')
+
+    assert paths == ['foo', 'food']
+
+
 def test_write_exclude_file_does_not_raise():
     temporary_file = flexmock(
         name='filename',
@@ -122,7 +140,8 @@ DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
 CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 
 
-def test_create_archive_should_call_borg_with_parameters():
+def test_create_archive_calls_borg_with_parameters():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND)
@@ -139,8 +158,9 @@ def test_create_archive_should_call_borg_with_parameters():
     )
 
 
-def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
+def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes'))
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
     insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
@@ -157,7 +177,8 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
     )
 
 
-def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
+def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
@@ -174,7 +195,8 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter
     )
 
 
-def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
+def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
@@ -191,7 +213,8 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete
     )
 
 
-def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
+def test_create_archive_with_compression_calls_borg_with_compression_parameters():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
@@ -208,7 +231,8 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param
     )
 
 
-def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
+def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
@@ -226,7 +250,8 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
     )
 
 
-def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
+def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
@@ -244,7 +269,8 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
     )
 
 
-def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
+def test_create_archive_with_umask_calls_borg_with_umask_parameters():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
@@ -262,6 +288,7 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
 
 
 def test_create_archive_with_source_directories_glob_expands():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
@@ -280,6 +307,7 @@ def test_create_archive_with_source_directories_glob_expands():
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo*'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
@@ -297,11 +325,11 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
     )
 
 
-def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
+def test_create_archive_with_glob_calls_borg_with_expanded_directories():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
-    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
         verbosity=None,
@@ -315,7 +343,8 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
     )
 
 
-def test_create_archive_with_archive_name_format_without_placeholders():
+def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
@@ -335,6 +364,7 @@ def test_create_archive_with_archive_name_format_without_placeholders():
 
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
     flexmock(module).should_receive('_write_exclude_file').and_return(None)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))
diff --git a/setup.py b/setup.py
index d3b8307e6..d23552be3 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.8'
+VERSION = '1.1.9.dev0'
 
 
 setup(

From bd196c1fb9317a87ea3e9911277d435261aebd26 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Sat, 9 Sep 2017 17:38:14 -0700
Subject: [PATCH 183/189] Removing "from __future__ import print_function".
 This isn't Python 2 anymore, Toto.

---
 borgmatic/commands/borgmatic.py       | 1 -
 borgmatic/commands/convert_config.py  | 1 -
 borgmatic/commands/generate_config.py | 1 -
 3 files changed, 3 deletions(-)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 2fdff34ad..b7c53a306 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 from argparse import ArgumentParser
 import os
 from subprocess import CalledProcessError
diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py
index 6a269c8e8..399a025b3 100644
--- a/borgmatic/commands/convert_config.py
+++ b/borgmatic/commands/convert_config.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 from argparse import ArgumentParser
 import os
 from subprocess import CalledProcessError
diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py
index 92aa797ea..b0776e0c6 100644
--- a/borgmatic/commands/generate_config.py
+++ b/borgmatic/commands/generate_config.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 from argparse import ArgumentParser
 import os
 from subprocess import CalledProcessError

From 86511deac4ca0afb6c357dbbf950c8f09fadd09e Mon Sep 17 00:00:00 2001
From: b3vis <b3vis@users.noreply.github.com>
Date: Thu, 26 Oct 2017 05:24:24 +0100
Subject: [PATCH 184/189] Added section about docker (#18)

---
 README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.md b/README.md
index 1e10d4455..6e370f2c6 100644
--- a/README.md
+++ b/README.md
@@ -66,6 +66,11 @@ To install borgmatic, run the following command to download and install it:
 Note that your pip binary may have a different name than "pip3". Make sure
 you're using Python 3, as borgmatic does not support Python 2.
 
+### Docker
+
+If you would like to run borgmatic within docker please take a look at 
+[b3vis/borgmatic](https://hub.docker.com/r/b3vis/borgmatic/) for more information.
+
 ## Configuration
 
 After you install borgmatic, generate a sample configuration file:

From f1a98d82c6e40754396cec4770effe74523773f0 Mon Sep 17 00:00:00 2001
From: Johannes Feichtner <Churro@users.noreply.github.com>
Date: Thu, 26 Oct 2017 06:38:27 +0200
Subject: [PATCH 185/189] #16, #38: Support for user-defined hooks before/after
 backup, or on error.

---
 AUTHORS                                |  1 +
 borgmatic/commands/borgmatic.py        | 38 ++++++++++++++++----------
 borgmatic/commands/hook.py             |  7 +++++
 borgmatic/config/generate.py           |  2 +-
 borgmatic/config/schema.yaml           | 25 +++++++++++++++++
 borgmatic/config/validate.py           |  2 +-
 borgmatic/tests/unit/borg/test_hook.py | 10 +++++++
 7 files changed, 68 insertions(+), 17 deletions(-)
 create mode 100644 borgmatic/commands/hook.py
 create mode 100644 borgmatic/tests/unit/borg/test_hook.py

diff --git a/AUTHORS b/AUTHORS
index bd99af8ca..81ae45e4a 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -5,3 +5,4 @@ Henning Schroeder: Copy editing
 Michele Lazzeri: Custom archive names
 Robin `ypid` Schneider: Support additional options of Borg
 Scott Squires: Custom archive names
+Johannes Feichtner: Support for user hooks
diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 2fdff34ad..e67fe88e7 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -5,6 +5,7 @@ from subprocess import CalledProcessError
 import sys
 
 from borgmatic.borg import check, create, prune
+from borgmatic.commands import hook
 from borgmatic.config import collect, convert, validate
 
 
@@ -84,26 +85,33 @@ def main():  # pragma: no cover
 
         for config_filename in config_filenames:
             config = validate.parse_configuration(config_filename, validate.schema_filename())
-            (location, storage, retention, consistency) = (
+            (location, storage, retention, consistency, hooks) = (
                 config.get(section_name, {})
-                for section_name in ('location', 'storage', 'retention', 'consistency')
+                for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
             )
             remote_path = location.get('remote_path')
 
-            create.initialize(storage)
+            try:
+                create.initialize(storage)
+                hook.execute_hook(hooks.get('before_backup'))
 
-            for repository in location['repositories']:
-                if args.prune:
-                    prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
-                if args.create:
-                    create.create_archive(
-                        args.verbosity,
-                        repository,
-                        location,
-                        storage,
-                    )
-                if args.check:
-                    check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
+                for repository in location['repositories']:
+                    if args.prune:
+                        prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+                    if args.create:
+                        create.create_archive(
+                            args.verbosity,
+                            repository,
+                            location,
+                            storage,
+                        )
+                    if args.check:
+                        check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
+
+                hook.execute_hook(hooks.get('after_backup'))
+            except (OSError, CalledProcessError):
+                hook.execute_hook(hooks.get('on_error'))
+                raise
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)
diff --git a/borgmatic/commands/hook.py b/borgmatic/commands/hook.py
new file mode 100644
index 000000000..4417dea05
--- /dev/null
+++ b/borgmatic/commands/hook.py
@@ -0,0 +1,7 @@
+import subprocess
+
+
+def execute_hook(commands):
+    if commands:
+        for cmd in commands:
+            subprocess.check_call(cmd, shell=True)
diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py
index 1a6d1a86e..2bfc3a259 100644
--- a/borgmatic/config/generate.py
+++ b/borgmatic/config/generate.py
@@ -24,7 +24,7 @@ def _schema_to_sample_configuration(schema, level=0):
     for each section based on the schema "desc" description.
     '''
     example = schema.get('example')
-    if example:
+    if example is not None:
         return example
 
     config = yaml.comments.CommentedMap([
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index d559ee3f3..60f6cdf5c 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -157,3 +157,28 @@ map:
                 desc: Restrict the number of checked archives to the last n. Applies only to the
                       "archives" check.
                 example: 3
+    hooks:
+        desc: |
+            Shell commands or scripts to execute before and after a backup or if an error has occurred.
+            IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
+            Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
+            prevent potential shell injection or privilege escalation.
+        map:
+            before_backup:
+                seq:
+                    - type: scalar
+                desc: List of one or more shell commands or scripts to execute before creating a backup.
+                example:
+                    - echo "`date` - Starting a backup job."
+            after_backup:
+                seq:
+                    - type: scalar
+                desc: List of one or more shell commands or scripts to execute after creating a backup.
+                example:
+                    - echo "`date` - Backup created."
+            on_error:
+                seq:
+                    - type: scalar
+                desc: List of one or more shell commands or scripts to execute in case an exception has occurred.
+                example:
+                    - echo "`date` - Error while creating a backup."
diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py
index 312517626..ff616341a 100644
--- a/borgmatic/config/validate.py
+++ b/borgmatic/config/validate.py
@@ -48,7 +48,7 @@ def parse_configuration(config_filename, schema_filename):
     # simply remove all examples before passing the schema to pykwalify.
     for section_name, section_schema in schema['map'].items():
         for field_name, field_schema in section_schema['map'].items():
-            field_schema.pop('example')
+            field_schema.pop('example', None)
 
     validator = pykwalify.core.Core(source_data=config, schema_data=schema)
     parsed_result = validator.validate(raise_exception=False)
diff --git a/borgmatic/tests/unit/borg/test_hook.py b/borgmatic/tests/unit/borg/test_hook.py
new file mode 100644
index 000000000..6aabc57b3
--- /dev/null
+++ b/borgmatic/tests/unit/borg/test_hook.py
@@ -0,0 +1,10 @@
+from flexmock import flexmock
+
+from borgmatic.commands import hook as module
+
+
+def test_execute_hook_invokes_each_command():
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(':', shell=True).once()
+
+    module.execute_hook([':'])

From 5c229639f0a3a581264400268b2aee2a776aa568 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Wed, 25 Oct 2017 21:47:33 -0700
Subject: [PATCH 186/189] Improve clarity of logging spew at high verbosity
 levels.

---
 NEWS                            |  2 ++
 borgmatic/commands/borgmatic.py | 12 ++++++++++++
 borgmatic/config/validate.py    |  2 ++
 borgmatic/verbosity.py          | 15 +++++++++++++++
 4 files changed, 31 insertions(+)

diff --git a/NEWS b/NEWS
index 102967909..cfac2f2a6 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,6 @@
 1.1.9.dev0
+ * #16, #38: Support for user-defined hooks before/after backup, or on error.
+ * #33: Improve clarity of logging spew at high verbosity levels.
  * #29: Support for using tilde in source directory path to reference home directory.
 
 1.1.8
diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index b7c53a306..28d1e0de8 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -1,10 +1,15 @@
 from argparse import ArgumentParser
+import logging
 import os
 from subprocess import CalledProcessError
 import sys
 
 from borgmatic.borg import check, create, prune
 from borgmatic.config import collect, convert, validate
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS, verbosity_to_log_level
+
+
+logger = logging.getLogger(__name__)
 
 
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
@@ -75,13 +80,17 @@ def parse_arguments(*arguments):
 def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
+        logging.basicConfig(level=verbosity_to_log_level(args.verbosity), format='%(message)s')
+
         config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+        logger.debug('Ensuring legacy configuration is upgraded')
         convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
 
         if len(config_filenames) == 0:
             raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths)))
 
         for config_filename in config_filenames:
+            logger.info('{}: Parsing configuration file'.format(config_filename))
             config = validate.parse_configuration(config_filename, validate.schema_filename())
             (location, storage, retention, consistency) = (
                 config.get(section_name, {})
@@ -93,8 +102,10 @@ def main():  # pragma: no cover
 
             for repository in location['repositories']:
                 if args.prune:
+                    logger.info('{}: Pruning archives'.format(repository))
                     prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
                 if args.create:
+                    logger.info('{}: Creating archive'.format(repository))
                     create.create_archive(
                         args.verbosity,
                         repository,
@@ -102,6 +113,7 @@ def main():  # pragma: no cover
                         storage,
                     )
                 if args.check:
+                    logger.info('{}: Running consistency checks'.format(repository))
                     check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py
index 312517626..0bac5964d 100644
--- a/borgmatic/config/validate.py
+++ b/borgmatic/config/validate.py
@@ -38,6 +38,8 @@ def parse_configuration(config_filename, schema_filename):
     Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
     have permissions to read the file, or Validation_error if the config does not match the schema.
     '''
+    logging.getLogger('pykwalify').setLevel(logging.ERROR)
+
     try:
         config = yaml.round_trip_load(open(config_filename))
         schema = yaml.round_trip_load(open(schema_filename))
diff --git a/borgmatic/verbosity.py b/borgmatic/verbosity.py
index 06dfc4c4e..13b1ecc02 100644
--- a/borgmatic/verbosity.py
+++ b/borgmatic/verbosity.py
@@ -1,2 +1,17 @@
+import logging
+
+
 VERBOSITY_SOME = 1
 VERBOSITY_LOTS = 2
+
+
+def verbosity_to_log_level(verbosity):
+    '''
+    Given a borgmatic verbosity value, return the corresponding Python log level.
+    '''
+    return {
+        VERBOSITY_SOME: logging.INFO,
+        VERBOSITY_LOTS: logging.DEBUG,
+    }.get(verbosity, logging.ERROR)
+
+

From cc78223164883465170e1066ba02010420fc9ddf Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Wed, 25 Oct 2017 21:58:02 -0700
Subject: [PATCH 187/189] Fixing inconsistent indentation.

---
 borgmatic/commands/borgmatic.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index 14d73bbba..c352b9d13 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -104,21 +104,21 @@ def main():  # pragma: no cover
                 create.initialize(storage)
                 hook.execute_hook(hooks.get('before_backup'))
 
-		        for repository in location['repositories']:
-		            if args.prune:
-		                logger.info('{}: Pruning archives'.format(repository))
-		                prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
-		            if args.create:
-		                logger.info('{}: Creating archive'.format(repository))
-		                create.create_archive(
-		                    args.verbosity,
-		                    repository,
-		                    location,
-		                    storage,
-		                )
-		            if args.check:
-		                logger.info('{}: Running consistency checks'.format(repository))
-		                check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
+                for repository in location['repositories']:
+                    if args.prune:
+                        logger.info('{}: Pruning archives'.format(repository))
+                        prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+                    if args.create:
+                        logger.info('{}: Creating archive'.format(repository))
+                        create.create_archive(
+                            args.verbosity,
+                            repository,
+                            location,
+                            storage,
+                        )
+                    if args.check:
+                        logger.info('{}: Running consistency checks'.format(repository))
+                        check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
 
                 hook.execute_hook(hooks.get('after_backup'))
             except (OSError, CalledProcessError):

From a09c9f248e7c25200a9d7102ead87a619f5fde83 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Wed, 25 Oct 2017 22:32:06 -0700
Subject: [PATCH 188/189] Adding logging to hook execution!

---
 borgmatic/commands/borgmatic.py               |  6 +--
 borgmatic/commands/hook.py                    | 21 ++++++++--
 .../tests/integration/config/test_validate.py | 42 +++++++++++++++++--
 borgmatic/tests/unit/borg/test_hook.py        | 14 ++++++-
 4 files changed, 71 insertions(+), 12 deletions(-)

diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py
index c352b9d13..8841b70eb 100644
--- a/borgmatic/commands/borgmatic.py
+++ b/borgmatic/commands/borgmatic.py
@@ -102,7 +102,7 @@ def main():  # pragma: no cover
 
             try:
                 create.initialize(storage)
-                hook.execute_hook(hooks.get('before_backup'))
+                hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
 
                 for repository in location['repositories']:
                     if args.prune:
@@ -120,9 +120,9 @@ def main():  # pragma: no cover
                         logger.info('{}: Running consistency checks'.format(repository))
                         check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
 
-                hook.execute_hook(hooks.get('after_backup'))
+                hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
             except (OSError, CalledProcessError):
-                hook.execute_hook(hooks.get('on_error'))
+                hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error')
                 raise
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
diff --git a/borgmatic/commands/hook.py b/borgmatic/commands/hook.py
index 4417dea05..c6f5a29cc 100644
--- a/borgmatic/commands/hook.py
+++ b/borgmatic/commands/hook.py
@@ -1,7 +1,20 @@
+import logging
 import subprocess
 
 
-def execute_hook(commands):
-    if commands:
-        for cmd in commands:
-            subprocess.check_call(cmd, shell=True)
+logger = logging.getLogger(__name__)
+
+
+def execute_hook(commands, config_filename, description):
+    if not commands:
+        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
+        return
+
+    if len(commands) == 1:
+        logger.info('{}: Running command for {} hook'.format(config_filename, description))
+    else:
+        logger.info('{}: Running {} commands for {} hook'.format(config_filename, len(commands), description))
+
+    for command in commands:
+        logger.debug('{}: Hook command: {}'.format(config_filename, command))
+        subprocess.check_call(command, shell=True)
diff --git a/borgmatic/tests/integration/config/test_validate.py b/borgmatic/tests/integration/config/test_validate.py
index 9b63ccc89..eec282cb6 100644
--- a/borgmatic/tests/integration/config/test_validate.py
+++ b/borgmatic/tests/integration/config/test_validate.py
@@ -15,13 +15,18 @@ def test_schema_filename_returns_plausable_path():
     assert schema_path.endswith('/schema.yaml')
 
 
-def mock_config_and_schema(config_yaml):
+def mock_config_and_schema(config_yaml, schema_yaml=None):
     '''
-    Set up mocks for the config config YAML string and the default schema so that the code under
-    test consumes them when parsing the configuration.
+    Set up mocks for the given config config YAML string and the schema YAML string, or the default
+    schema if no schema is provided. The idea is that that the code under test consumes these mocks
+    when parsing the configuration.
     '''
     config_stream = io.StringIO(config_yaml)
-    schema_stream = open(module.schema_filename())
+    if schema_yaml is None:
+        schema_stream = open(module.schema_filename())
+    else:
+        schema_stream = io.StringIO(schema_yaml)
+
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('config.yaml').and_return(config_stream)
     builtins.should_receive('open').with_args('schema.yaml').and_return(schema_stream)
@@ -81,6 +86,35 @@ def test_parse_configuration_passes_through_quoted_punctuation():
     }
 
 
+def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+        ''',
+        '''
+        map:
+            location:
+                required: true
+                map:
+                    source_directories:
+                        required: true
+                        seq:
+                            - type: scalar
+                    repositories:
+                        required: true
+                        seq:
+                            - type: scalar
+        '''
+    )
+
+    module.parse_configuration('config.yaml', 'schema.yaml')
+
+
 def test_parse_configuration_raises_for_missing_config_file():
     with pytest.raises(FileNotFoundError):
         module.parse_configuration('config.yaml', 'schema.yaml')
diff --git a/borgmatic/tests/unit/borg/test_hook.py b/borgmatic/tests/unit/borg/test_hook.py
index 6aabc57b3..f87edda6b 100644
--- a/borgmatic/tests/unit/borg/test_hook.py
+++ b/borgmatic/tests/unit/borg/test_hook.py
@@ -7,4 +7,16 @@ def test_execute_hook_invokes_each_command():
     subprocess = flexmock(module.subprocess)
     subprocess.should_receive('check_call').with_args(':', shell=True).once()
 
-    module.execute_hook([':'])
+    module.execute_hook([':'], 'config.yaml', 'pre-backup')
+
+
+def test_execute_hook_with_multiple_commands_invokes_each_command():
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(':', shell=True).once()
+    subprocess.should_receive('check_call').with_args('true', shell=True).once()
+
+    module.execute_hook([':', 'true'], 'config.yaml', 'pre-backup')
+
+
+def test_execute_hook_with_empty_commands_does_not_raise():
+    module.execute_hook([], 'config.yaml', 'post-backup')

From 2ae8ac2947bfd651a43af042c5042b414c4c50c7 Mon Sep 17 00:00:00 2001
From: Dan Helfman <witten@torsion.org>
Date: Wed, 25 Oct 2017 22:36:23 -0700
Subject: [PATCH 189/189] Add tests for verbosity mapping.

---
 borgmatic/tests/unit/test_verbosity.py | 11 +++++++++++
 borgmatic/verbosity.py                 |  2 --
 2 files changed, 11 insertions(+), 2 deletions(-)
 create mode 100644 borgmatic/tests/unit/test_verbosity.py

diff --git a/borgmatic/tests/unit/test_verbosity.py b/borgmatic/tests/unit/test_verbosity.py
new file mode 100644
index 000000000..685aed07a
--- /dev/null
+++ b/borgmatic/tests/unit/test_verbosity.py
@@ -0,0 +1,11 @@
+import logging
+
+from borgmatic import verbosity as module
+
+
+def test_verbosity_to_log_level_maps_known_verbosity_to_log_level():
+    assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO
+
+
+def test_verbosity_to_log_level_maps_unknown_verbosity_to_error_level():
+    assert module.verbosity_to_log_level('my pants') == logging.ERROR
diff --git a/borgmatic/verbosity.py b/borgmatic/verbosity.py
index 13b1ecc02..e1ea796dd 100644
--- a/borgmatic/verbosity.py
+++ b/borgmatic/verbosity.py
@@ -13,5 +13,3 @@ def verbosity_to_log_level(verbosity):
         VERBOSITY_SOME: logging.INFO,
         VERBOSITY_LOTS: logging.DEBUG,
     }.get(verbosity, logging.ERROR)
-
-