Database dump hooks documentation (#225).

.eleventy.js

@@ -13,10 +13,11 @@ module.exports = function(eleventyConfig) {
         html: true,
         breaks: false,
         linkify: true,
-        // Replace links to .md files with links to directories. This allows unparsed Markdown links
-        // to work on GitHub, while rendered links elsewhere also work.
         replaceLink: function (link, env) {
-            return link.replace(/\.md$/, '/');
+            if (process.env.NODE_ENV == "production") {
+                return link;
+            }
+            return link.replace('https://torsion.org/borgmatic/', 'http://localhost:8080/');
         }
     };
     let markdownItAnchorOptions = {

.gitignore

@@ -5,6 +5,7 @@
 .coverage
 .pytest_cache
 .tox
+__pycache__
 build/
 dist/
 pip-wheel-metadata/

NEWS

@@ -1,3 +1,20 @@
+1.4.0
+ * #225: Database dump hooks for PostgreSQL, so you can easily dump your databases before backups
+   run.
+ * #230: Rename "borgmatic list --pattern-from" flag to "--patterns-from" to match Borg.
+
+1.3.26
+ * #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
+   (non-checkpoint) archives.
+
+1.3.25
+ * #223: Dead man's switch to detect when backups start failing silently, implemented via
+   healthchecks.io hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
+ * Documentation on monitoring and alerting options for borgmatic backups:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/
+ * Automatically rewrite links when developing on documentation locally.
+
 1.3.24
  * #86: Add "borgmatic list --successful" flag to only list successful (non-checkpoint) archives.
  * Add a suggestion form to all documentation pages, so users can submit ideas for improving the

README.md

@@ -41,10 +41,18 @@ retention:
     keep_monthly: 6
 
 consistency:
-    # List of consistency checks to run: "repository", "archives", or both.
+    # List of consistency checks to run: "repository", "archives", etc.
     checks:
         - repository
         - archives
+
+hooks:
+    # Preparation scripts to run, databases to dump, and monitoring to perform.
+    before_backup:
+        - prepare-for-backup.sh
+    postgresql_databases:
+        - name: users
+    healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
 ```
 
 borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
@@ -63,7 +71,9 @@ href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
  * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
  * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
  * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
  * [Restore a backup](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/)
+ * [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
  * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
  * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
  * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
@@ -116,8 +126,3 @@ your thing. In general, contributions are very welcome. We don't bite!
 Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
-
-<script>
-  var links = document.getElementsByClassName("referral");
-  links[Math.floor(Math.random() * links.length)].style.display = "none";
-</script>

borgmatic/borg/create.py

@@ -60,8 +60,8 @@ def _write_pattern_file(patterns=None):
 
 def _make_pattern_flags(location_config, pattern_filename=None):
     '''
-    Given a location config dict with a potential pattern_from option, and a filename containing any
-    additional patterns, return the corresponding Borg flags for those files as a tuple.
+    Given a location config dict with a potential patterns_from option, and a filename containing
+    any additional patterns, return the corresponding Borg flags for those files as a tuple.
     '''
     pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
         (pattern_filename,) if pattern_filename else ()
@@ -94,6 +94,20 @@ def _make_exclude_flags(location_config, exclude_filename=None):
     return exclude_from_flags + caches_flag + if_present_flags
 
 
+BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
+
+
+def borgmatic_source_directories():
+    '''
+    Return a list of borgmatic-specific source directories used for state like database backups.
+    '''
+    return (
+        [BORGMATIC_SOURCE_DIRECTORY]
+        if os.path.exists(os.path.expanduser(BORGMATIC_SOURCE_DIRECTORY))
+        else []
+    )
+
+
 def create_archive(
     dry_run,
     repository,
@@ -109,7 +123,9 @@ def create_archive(
     Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
     storage config dict, create a Borg archive and return Borg's JSON output (if any).
     '''
-    sources = _expand_directories(location_config['source_directories'])
+    sources = _expand_directories(
+        location_config['source_directories'] + borgmatic_source_directories()
+    )
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
     exclude_file = _write_pattern_file(

borgmatic/borg/list.py

@@ -6,8 +6,9 @@ from borgmatic.execute import execute_command
 logger = logging.getLogger(__name__)
 
 
-# A hack to convince Borg to exclude archives ending in ".checkpoint".
-BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[!.][!c][!h][!e][!c][!k][!p][!o][!i][!n][!t]'
+# A hack to convince Borg to exclude archives ending in ".checkpoint". This assumes that a
+# non-checkpoint archive name ends in a digit (e.g. from a timestamp).
+BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[0123456789]'
 
 
 def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):

borgmatic/commands/arguments.py

@@ -339,7 +339,7 @@ def parse_arguments(*unparsed_arguments):
     )
     list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
     list_group.add_argument(
-        '--pattern-from',
+        '--patterns-from',
         metavar='FILENAME',
         help='Include or exclude paths matching patterns from pattern file, one per line',
     )

borgmatic/commands/borgmatic.py

@@ -8,7 +8,6 @@ from subprocess import CalledProcessError
 import colorama
 import pkg_resources
 
-from borgmatic import hook
 from borgmatic.borg import check as borg_check
 from borgmatic.borg import create as borg_create
 from borgmatic.borg import environment as borg_environment
@@ -19,6 +18,7 @@ from borgmatic.borg import list as borg_list
 from borgmatic.borg import prune as borg_prune
 from borgmatic.commands.arguments import parse_arguments
 from borgmatic.config import checks, collect, convert, validate
+from borgmatic.hooks import command, healthchecks, postgresql
 from borgmatic.logger import configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
@@ -53,13 +53,19 @@ def run_configuration(config_filename, config, arguments):
 
     if 'create' in arguments:
         try:
-            hook.execute_hook(
+            command.execute_hook(
                 hooks.get('before_backup'),
                 hooks.get('umask'),
                 config_filename,
                 'pre-backup',
                 global_arguments.dry_run,
             )
+            postgresql.dump_databases(
+                hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run
+            )
+            healthchecks.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
+            )
         except (OSError, CalledProcessError) as error:
             encountered_error = error
             yield from make_error_log_records(
@@ -88,13 +94,19 @@ def run_configuration(config_filename, config, arguments):
 
     if 'create' in arguments and not encountered_error:
         try:
-            hook.execute_hook(
+            command.execute_hook(
                 hooks.get('after_backup'),
                 hooks.get('umask'),
                 config_filename,
                 'post-backup',
                 global_arguments.dry_run,
             )
+            postgresql.remove_database_dumps(
+                hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run
+            )
+            healthchecks.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run
+            )
         except (OSError, CalledProcessError) as error:
             encountered_error = error
             yield from make_error_log_records(
@@ -103,7 +115,7 @@ def run_configuration(config_filename, config, arguments):
 
     if encountered_error:
         try:
-            hook.execute_hook(
+            command.execute_hook(
                 hooks.get('on_error'),
                 hooks.get('umask'),
                 config_filename,
@@ -113,6 +125,9 @@ def run_configuration(config_filename, config, arguments):
                 error=encountered_error,
                 output=getattr(encountered_error, 'output', ''),
             )
+            healthchecks.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail'
+            )
         except (OSError, CalledProcessError) as error:
             yield from make_error_log_records(
                 '{}: Error running on-error hook'.format(config_filename), error
@@ -330,7 +345,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
         try:
             for config_filename, config in configs.items():
                 hooks = config.get('hooks', {})
-                hook.execute_hook(
+                command.execute_hook(
                     hooks.get('before_everything'),
                     hooks.get('umask'),
                     config_filename,
@@ -370,7 +385,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
         try:
             for config_filename, config in configs.items():
                 hooks = config.get('hooks', {})
-                hook.execute_hook(
+                command.execute_hook(
                     hooks.get('after_everything'),
                     hooks.get('umask'),
                     config_filename,

borgmatic/config/convert.py

@@ -54,10 +54,10 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
 
     # Add comments to each section, and then add comments to the fields in each section.
-    generate.add_comments_to_configuration(destination_config, schema)
+    generate.add_comments_to_configuration_map(destination_config, schema)
 
     for section_name, section_config in destination_config.items():
-        generate.add_comments_to_configuration(
+        generate.add_comments_to_configuration_map(
             section_config, schema['map'][section_name], indent=generate.INDENT
         )
 

borgmatic/config/generate.py

@@ -1,8 +1,11 @@
+import io
 import os
+import re
 
 from ruamel import yaml
 
 INDENT = 4
+SEQUENCE_INDENT = 2
 
 
 def _insert_newline_before_comment(config, field_name):
@@ -15,7 +18,7 @@ def _insert_newline_before_comment(config, field_name):
     )
 
 
-def _schema_to_sample_configuration(schema, level=0):
+def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     '''
     Given a loaded configuration schema, generate and return sample config for it. Include comments
     for each section based on the schema "desc" description.
@@ -24,14 +27,29 @@ def _schema_to_sample_configuration(schema, level=0):
     if example is not None:
         return example
 
-    config = yaml.comments.CommentedMap(
-        [
-            (section_name, _schema_to_sample_configuration(section_schema, level + 1))
-            for section_name, section_schema in schema['map'].items()
-        ]
-    )
-
-    add_comments_to_configuration(config, schema, indent=(level * INDENT))
+    if 'seq' in schema:
+        config = yaml.comments.CommentedSeq(
+            [
+                _schema_to_sample_configuration(item_schema, level, parent_is_sequence=True)
+                for item_schema in schema['seq']
+            ]
+        )
+        add_comments_to_configuration_sequence(
+            config, schema, indent=(level * INDENT) + SEQUENCE_INDENT
+        )
+    elif 'map' in schema:
+        config = yaml.comments.CommentedMap(
+            [
+                (section_name, _schema_to_sample_configuration(section_schema, level + 1))
+                for section_name, section_schema in schema['map'].items()
+            ]
+        )
+        indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
+        add_comments_to_configuration_map(
+            config, schema, indent=indent, skip_first=parent_is_sequence
+        )
+    else:
+        raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema))
 
     return config
 
@@ -42,13 +60,12 @@ def _comment_out_line(line):
     if not stripped_line or stripped_line.startswith('#'):
         return line
 
-    # Comment out the names of optional sections.
-    one_indent = ' ' * INDENT
-    if not line.startswith(one_indent):
-        return '# ' + line
+    # Comment out the names of optional sections, inserting the '#' after any indent for aesthetics.
+    matches = re.match(r'(\s*)', line)
+    indent_spaces = matches.group(0) if matches else ''
+    count_indent_spaces = len(indent_spaces)
 
-    # Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics.
-    return '# '.join((one_indent, line[INDENT:]))
+    return '# '.join((indent_spaces, line[count_indent_spaces:]))
 
 
 REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
@@ -90,7 +107,12 @@ def _render_configuration(config):
     '''
     Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
     '''
-    return yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)
+    dumper = yaml.YAML()
+    dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
+    rendered = io.StringIO()
+    dumper.dump(config, rendered)
+
+    return rendered.getvalue()
 
 
 def write_configuration(config_filename, rendered_config, mode=0o600):
@@ -112,13 +134,49 @@ def write_configuration(config_filename, rendered_config, mode=0o600):
     os.chmod(config_filename, mode)
 
 
-def add_comments_to_configuration(config, schema, indent=0):
+def add_comments_to_configuration_sequence(config, schema, indent=0):
+    '''
+    If the given config sequence's items are maps, then mine the schema for the description of the
+    map's first item, and slap that atop the sequence. Indent the comment the given number of
+    characters.
+
+    Doing this for sequences of maps results in nice comments that look like:
+
+    ```
+    things:
+          # First key description. Added by this function.
+        - key: foo
+          # Second key description. Added by add_comments_to_configuration_map().
+          other: bar
+    ```
+    '''
+    if 'map' not in schema['seq'][0]:
+        return
+
+    for field_name in config[0].keys():
+        field_schema = schema['seq'][0]['map'].get(field_name, {})
+        description = field_schema.get('desc')
+
+        # No description to use? Skip it.
+        if not field_schema or not description:
+            return
+
+        config[0].yaml_set_start_comment(description, indent=indent)
+
+        # We only want the first key's description here, as the rest of the keys get commented by
+        # add_comments_to_configuration_map().
+        return
+
+
+def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False):
     '''
     Using descriptions from a schema as a source, add those descriptions as comments to the given
-    config before each field. This function only adds comments for the top-most config map level.
-    Indent the comment the given number of characters.
+    config mapping, before each field. Indent the comment the given number of characters.
     '''
     for index, field_name in enumerate(config.keys()):
+        if skip_first and index == 0:
+            continue
+
         field_schema = schema['map'].get(field_name, {})
         description = field_schema.get('desc')
 
@@ -127,6 +185,7 @@ def add_comments_to_configuration(config, schema, indent=0):
             continue
 
         config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
+
         if index > 0:
             _insert_newline_before_comment(config, field_name)
 

borgmatic/config/schema.yaml

@@ -337,8 +337,8 @@ map:
                 example: false
     hooks:
         desc: |
-            Shell commands or scripts to execute at various points during a borgmatic run.
-            IMPORTANT: All provided commands and scripts are executed with user permissions of
+            Shell commands, scripts, or integrations to execute at various points during a borgmatic
+            run. IMPORTANT: All provided commands and scripts are executed with user permissions of
             borgmatic. Do not forget to set secure permissions on this configuration file (chmod
             0600) as well as on any script called from a hook (chmod 0700) to prevent potential
             shell injection or privilege escalation.
@@ -363,10 +363,73 @@ map:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute when an exception occurs
-                    during a backup or when running a before_backup or after_backup hook.
+                    List of one or more shell commands or scripts to execute when an exception
+                    occurs during a backup or when running a before_backup or after_backup hook.
                 example:
                     - echo "Error while creating a backup or running a backup hook."
+            postgresql_databases:
+                seq:
+                    - map:
+                        name:
+                            required: true
+                            type: str
+                            desc: |
+                                Database name (required if using this hook). Or "all" to dump all
+                                databases on the host.
+                            example: users
+                        hostname:
+                            type: str
+                            desc: |
+                                Database hostname to connect to. Defaults to connecting via local
+                                Unix socket.
+                            example: database.example.org
+                        port:
+                            type: int
+                            desc: Port to connect to. Defaults to 5432.
+                            example: 5433
+                        username:
+                            type: str
+                            desc: |
+                                Username with which to connect to the database. Defaults to the
+                                username of the current user. You probably want to specify the
+                                "postgres" superuser here when the database name is "all".
+                            example: dbuser
+                        password:
+                            type: str
+                            desc: |
+                                Password with which to connect to the database. Omitting a password
+                                will only work if PostgreSQL is configured to trust the configured
+                                username without a password, or you create a ~/.pgpass file.
+                            example: trustsome1
+                        format:
+                            type: str
+                            enum: ['plain', 'custom', 'directory', 'tar']
+                            desc: |
+                                Database dump output format. One of "plain", "custom", "directory",
+                                or "tar". Defaults to "custom" (unlike raw pg_dump). See
+                                https://www.postgresql.org/docs/current/app-pgdump.html for details.
+                                Note that format is ignored when the database name is "all".
+                            example: directory
+                        options:
+                            type: str
+                            desc: |
+                                Additional pg_dump/pg_dumpall options to pass directly to the dump
+                                command, without performing any validation on them. See
+                                https://www.postgresql.org/docs/current/app-pgdump.html for details.
+                            example: --role=someone
+                desc: |
+                    List of one or more PostgreSQL databases to dump before creating a backup,
+                    run once per configuration file. The database dumps are added to your source
+                    directories at runtime, backed up, and then removed afterwards. Requires
+                    pg_dump/pg_dumpall/pg_restore commands. See
+                    https://www.postgresql.org/docs/current/app-pgdump.html for details.
+            healthchecks:
+                type: str
+                desc: |
+                    Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
+                    Create an account at https://healthchecks.io if you'd like to use this service.
+                example:
+                    https://hc-ping.com/your-uuid-here
             before_everything:
                 seq:
                     - type: str

borgmatic/config/validate.py

@@ -64,6 +64,23 @@ def apply_logical_validation(config_filename, parsed_configuration):
             )
 
 
+def remove_examples(schema):
+    '''
+    pykwalify gets angry if the example field is not a string. So rather than bend to its will,
+    remove all examples from the given schema before passing the schema to pykwalify.
+    '''
+    if 'map' in schema:
+        for item_name, item_schema in schema['map'].items():
+            item_schema.pop('example', None)
+            remove_examples(item_schema)
+    elif 'seq' in schema:
+        for item_schema in schema['seq']:
+            item_schema.pop('example', None)
+            remove_examples(item_schema)
+
+    return schema
+
+
 def parse_configuration(config_filename, schema_filename):
     '''
     Given the path to a config filename in YAML format and the path to a schema filename in
@@ -84,13 +101,7 @@ def parse_configuration(config_filename, schema_filename):
     except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
-    # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
-    # remove all examples before passing the schema to pykwalify.
-    for section_name, section_schema in schema['map'].items():
-        for field_name, field_schema in section_schema['map'].items():
-            field_schema.pop('example', None)
-
-    validator = pykwalify.core.Core(source_data=config, schema_data=schema)
+    validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
     parsed_result = validator.validate(raise_exception=False)
 
     if validator.validation_errors:

borgmatic/execute.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import subprocess
 
 logger = logging.getLogger(__name__)
@@ -8,10 +9,17 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25
 BORG_ERROR_EXIT_CODE = 2
 
 
-def execute_and_log_output(full_command, output_log_level, shell):
+def borg_command(full_command):
+    '''
+    Return True if this is a Borg command, or False if it's some other command.
+    '''
+    return 'borg' in full_command[0]
+
+
+def execute_and_log_output(full_command, output_log_level, shell, environment):
     last_lines = []
     process = subprocess.Popen(
-        full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell
+        full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, env=environment
     )
 
     while process.poll() is None:
@@ -33,9 +41,11 @@ def execute_and_log_output(full_command, output_log_level, shell):
 
     exit_code = process.poll()
 
-    # If shell is True, assume we're running something other than Borg and should treat all non-zero
-    # exit codes as errors.
-    error = bool(exit_code != 0) if shell else bool(exit_code >= BORG_ERROR_EXIT_CODE)
+    # If we're running something other than Borg, treat all non-zero exit codes as errors.
+    if borg_command(full_command):
+        error = bool(exit_code >= BORG_ERROR_EXIT_CODE)
+    else:
+        error = bool(exit_code != 0)
 
     if error:
         # If an error occurs, include its output in the raised exception so that we don't
@@ -48,21 +58,25 @@ def execute_and_log_output(full_command, output_log_level, shell):
         )
 
 
-def execute_command(full_command, output_log_level=logging.INFO, shell=False):
+def execute_command(
+    full_command, output_log_level=logging.INFO, shell=False, extra_environment=None
+):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
     given log level. If output log level is None, instead capture and return the output. If
-    shell is True, execute the command within a shell.
+    shell is True, execute the command within a shell. If an extra environment dict is given, then
+    use it to augment the current environment, and pass the result into the command.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
     logger.debug(' '.join(full_command))
+    environment = {**os.environ, **extra_environment} if extra_environment else None
 
     if output_log_level is None:
-        output = subprocess.check_output(full_command, shell=shell)
+        output = subprocess.check_output(full_command, shell=shell, env=environment)
         return output.decode() if output is not None else None
     else:
-        execute_and_log_output(full_command, output_log_level, shell=shell)
+        execute_and_log_output(full_command, output_log_level, shell=shell, environment=environment)
 
 
 def execute_command_without_capture(full_command):

borgmatic/hooks/healthchecks.py

@@ -0,0 +1,36 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
+    '''
+    Ping the given healthchecks.io URL or UUID, appending the append string if any. Use the given
+    configuration filename in any log entries. If this is a dry run, then don't actually ping
+    anything.
+    '''
+    if not ping_url_or_uuid:
+        logger.debug('{}: No healthchecks hook set'.format(config_filename))
+        return
+
+    ping_url = (
+        ping_url_or_uuid
+        if ping_url_or_uuid.startswith('http')
+        else 'https://hc-ping.com/{}'.format(ping_url_or_uuid)
+    )
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+
+    if append:
+        ping_url = '{}/{}'.format(ping_url, append)
+
+    logger.info(
+        '{}: Pinging healthchecks.io{}{}'.format(
+            config_filename, ' ' + append if append else '', dry_run_label
+        )
+    )
+    logger.debug('{}: Using healthchecks.io ping URL {}'.format(config_filename, ping_url))
+
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    requests.get(ping_url)

borgmatic/hooks/postgresql.py

@@ -0,0 +1,88 @@
+import logging
+import os
+
+from borgmatic.execute import execute_command
+
+DUMP_PATH = '~/.borgmatic/postgresql_databases'
+logger = logging.getLogger(__name__)
+
+
+def dump_databases(databases, config_filename, dry_run):
+    '''
+    Dump the given PostgreSQL databases to disk. The databases are supplied as a sequence of dicts,
+    one dict describing each database as per the configuration schema. Use the given configuration
+    filename in any log entries. If this is a dry run, then don't actually dump anything.
+    '''
+    if not databases:
+        logger.debug('{}: No PostgreSQL databases configured'.format(config_filename))
+        return
+
+    dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+
+    logger.info('{}: Dumping PostgreSQL databases{}'.format(config_filename, dry_run_label))
+
+    for database in databases:
+        if os.path.sep in database['name']:
+            raise ValueError('Invalid database name {}'.format(database['name']))
+
+        dump_path = os.path.join(
+            os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost')
+        )
+        name = database['name']
+        all_databases = bool(name == 'all')
+        command = (
+            ('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean')
+            + ('--file', os.path.join(dump_path, name))
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--username', database['username']) if 'username' in database else ())
+            + (() if all_databases else ('--format', database.get('format', 'custom')))
+            + (tuple(database['options'].split(' ')) if 'options' in database else ())
+            + (() if all_databases else (name,))
+        )
+        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+
+        logger.debug(
+            '{}: Dumping PostgreSQL database {}{}'.format(config_filename, name, dry_run_label)
+        )
+        if not dry_run:
+            os.makedirs(dump_path, mode=0o700, exist_ok=True)
+            execute_command(command, extra_environment=extra_environment)
+
+
+def remove_database_dumps(databases, config_filename, dry_run):
+    '''
+    Remove the database dumps for the given databases. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the given
+    configuration filename in any log entries. If this is a dry run, then don't actually remove
+    anything.
+    '''
+    if not databases:
+        logger.debug('{}: No PostgreSQL databases configured'.format(config_filename))
+        return
+
+    dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
+
+    logger.info('{}: Removing PostgreSQL database dumps{}'.format(config_filename, dry_run_label))
+
+    for database in databases:
+        if os.path.sep in database['name']:
+            raise ValueError('Invalid database name {}'.format(database['name']))
+
+        name = database['name']
+        dump_path = os.path.join(
+            os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost')
+        )
+        dump_filename = os.path.join(dump_path, name)
+
+        logger.debug(
+            '{}: Remove PostgreSQL database dump {} from {}{}'.format(
+                config_filename, name, dump_filename, dry_run_label
+            )
+        )
+        if dry_run:
+            continue
+
+        os.remove(dump_filename)
+        if len(os.listdir(dump_path)) == 0:
+            os.rmdir(dump_path)

docs/Dockerfile

@@ -9,6 +9,8 @@ RUN borgmatic --help > /command-line.txt \
 
 FROM node:12.10.0-alpine as html
 
+ARG ENVIRONMENT=production
+
 WORKDIR /source
 
 RUN npm install @11ty/eleventy \
@@ -20,7 +22,7 @@ RUN npm install @11ty/eleventy \
 COPY --from=borgmatic /etc/borgmatic/config.yaml /source/docs/_includes/borgmatic/config.yaml
 COPY --from=borgmatic /command-line.txt /source/docs/_includes/borgmatic/command-line.txt
 COPY . /source
-RUN npx eleventy --input=/source/docs --output=/output/docs \
+RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
 FROM nginx:1.16.1-alpine

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -1,23 +1,26 @@
 ---
-title: Add preparation and cleanup steps to backups
+title: How to add preparation and cleanup steps to backups
 ---
 ## Preparation and cleanup hooks
 
 If you find yourself performing prepraration tasks before your backup runs, or
-cleanup work afterwards, borgmatic hooks may be of interest. Hooks are
-shell commands that borgmatic executes for you at various points, and they're
-configured in the `hooks` section of your configuration file.
+cleanup work afterwards, borgmatic hooks may be of interest. Hooks are shell
+commands that borgmatic executes for you at various points, and they're
+configured in the `hooks` section of your configuration file. But if you're
+looking to backup a database, it's probably easier to use the [database backup
+feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
+instead.
 
-For instance, you can specify `before_backup` hooks to dump a database to file
-before backing it up, and specify `after_backup` hooks to delete the temporary
-file afterwards. Here's an example:
+You can specify `before_backup` hooks to perform preparation steps before
+running backups, and specify `after_backup` hooks to perform cleanup steps
+afterwards. Here's an example:
 
 ```yaml
 hooks:
     before_backup:
-        - dump-a-database /to/file.sql
+        - mount /some/filesystem
     after_backup:
-        - rm /to/file.sql
+        - umount /some/filesystem
 ```
 
 The `before_backup` and `after_backup` hooks each run once per configuration
@@ -48,15 +51,15 @@ a backup or a backup hook, but not if an error occurs during a
 `before_everything` hook.
 
 borgmatic also runs `on_error` hooks if an error occurs, either when creating
-a backup or running a backup hook. See the [error alerting
-documentation](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+a backup or running a backup hook. See the [monitoring and alerting
+documentation](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
 for more information.
 
 ## Hook output
 
 Any output produced by your hooks shows up both at the console and in syslog
 (when run in a non-interactive console). For more information, read about <a
-href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md">inspecting
+href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/">inspecting
 your backups</a>.
 
 ## Security
@@ -70,6 +73,7 @@ invoked by hooks.
 
 ## Related documentation
 
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
- * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
- * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)

docs/how-to/backup-your-databases.md

@@ -0,0 +1,81 @@
+---
+title: How to backup your databases
+---
+## Database dump hooks
+
+If you want to backup a database, it's best practice with most database
+systems to backup an exported database dump, rather than backing up your
+database's internal file storage. That's because the internal storage can
+change while you're reading from it. In contrast, a database dump creates a
+consistent snapshot that is more suited for backups.
+
+Fortunately, borgmatic includes built-in support for creating database dumps
+prior to running backups. For example, here is everything you need to dump and
+backup a couple of local PostgreSQL databases:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: users
+        - name: orders
+```
+
+Prior to each backup, borgmatic dumps each configured database to a file
+(located in `~/.borgmatic/`) and includes it in the backup. After the backup
+completes, borgmatic removes the database dump files to recover disk space.
+
+Here's a more involved example that connects to a remote database:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: users
+          hostname: database.example.org
+          port: 5433
+          username: dbuser
+          password: trustsome1
+          format: tar
+          options: "--role=someone"
+```
+
+If you want to dump all databases on a host, use `all` for the database name:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: all
+```
+
+Note that you may need to use a `username` of the `postgres` superuser for
+this to work.
+
+## Supported databases
+
+As of now, borgmatic only supports PostgreSQL databases directly. But see
+below about general-purpose preparation and cleanup hooks as a work-around
+with other database systems. Also, please [file a
+ticket](https://torsion.org/borgmatic/#issues) for additional database systems
+that you'd like supported.
+
+## Database restoration
+
+borgmatic does not yet perform integrated database restoration when you
+[restore a backup](http://localhost:8080/docs/how-to/restore-a-backup/), but
+that feature is coming in a future release. In the meantime, you can restore
+a database manually after restoring a dump file in the `~/.borgmatic` path.
+
+## Preparation and cleanup hooks
+
+If this database integration is too limited for needs, borgmatic also supports
+general-purpose [preparation and cleanup
+hooks](https://torsion.org/borgmatic/docs/how-to/set-up-backups/). These
+hooks allows you to trigger arbitrary commands or scripts before and after
+backups. So if necessary, you can use these hooks to create database dumps
+with any database system.
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Restore a backup](http://localhost:8080/docs/how-to/restore-a-backup/)

docs/how-to/deal-with-very-large-backups.md

@@ -106,4 +106,4 @@ backups.
 
 ## Related documentation
 
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)

docs/how-to/develop-on-borgmatic.md

@@ -109,4 +109,4 @@ also linked from the commits list on each pull request.
 
 ## Related documentation
 
- * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)

docs/how-to/inspect-your-backups.md

@@ -22,7 +22,7 @@ borgmatic --verbosity 2
 
 ## Backup summary
 
-If you're less concerned with progress during a backup, and you just want to
+If you're less concerned with progress during a backup, and you only want to
 see the summary of archive statistics at the end, you can use the stats
 option when performing a backup:
 
@@ -83,78 +83,10 @@ Note that the [sample borgmatic systemd service
 file](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#systemd)
 already has this rate limit disabled.
 
-## Error alerting
-
-When an error occurs during a backup, borgmatic can run configurable shell
-commands to fire off custom error notifications or take other actions, so you
-can get alerted as soon as something goes wrong. Here's a not-so-useful
-example:
-
-```yaml
-hooks:
-    on_error:
-        - echo "Error while creating a backup or running a backup hook."
-```
-
-The `on_error` hook supports interpolating particular runtime variables into
-the hook command. Here's an example that assumes you provide a separate shell
-script to handle the alerting:
-
-```yaml
-hooks:
-    on_error:
-        - send-text-message.sh "{configuration_filename}" "{repository}"
-```
-
-In this example, when the error occurs, borgmatic interpolates a few runtime
-values into the hook command: the borgmatic configuration filename, and the
-path of the repository. Here's the full set of supported variables you can use
-here:
-
- * `configuration_filename`: borgmatic configuration filename in which the
-   error occurred
- * `repository`: path of the repository in which the error occurred (may be
-   blank if the error occurs in a hook)
- * `error`: the error message itself
- * `output`: output of the command that failed (may be blank if an error
-   occurred without running a command)
-
-Note that borgmatic does not run `on_error` hooks if an error occurs within a
-`before_everything` or `after_everything` hook. For more about hooks, see the
-[borgmatic hooks
-documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md),
-especially the security information.
-
-
-## Scripting borgmatic
-
-To consume the output of borgmatic in other software, you can include an
-optional `--json` flag with `create`, `list`, or `info` to get the output
-formatted as JSON.
-
-Note that when you specify the `--json` flag, Borg's other non-JSON output is
-suppressed so as not to interfere with the captured JSON. Also note that JSON
-output only shows up at the console, and not in syslog.
-
-### Successful backups
-
-`borgmatic list` includes support for a `--successful` flag that only lists
-successful (non-checkpoint) backups. Combined with a built-in Borg flag like
-`--last`, you can list the last successful backup for use in your monitoring
-scripts. Here's an example combined with `--json`:
-
-```bash
-borgmatic list --successful --last 1 --json
-```
-
-Note that this particular combination will only work if you've got a single
-backup "series" in your repository. If you're instead backing up, say, from
-multiple different hosts into a single repository, then you'll need to get
-fancier with your archive listing. See `borg list --help` for more flags.
-
 
 ## Related documentation
 
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
- * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md)
- * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)

docs/how-to/make-per-application-backups.md

@@ -112,4 +112,4 @@ directly, please see the section above about standard includes.
 
 ## Related documentation
 
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)

docs/how-to/monitor-your-backups.md

@@ -0,0 +1,158 @@
+---
+title: How to monitor your backups
+---
+
+## Monitoring and alerting
+
+Having backups is great, but they won't do you a lot of good unless you have
+confidence that they're running on a regular basis. That's where monitoring
+and alerting comes in.
+
+There are several different ways you can monitor your backups and find out
+whether they're succeeding. Which of these you choose to do is up to you and
+your particular infrastructure:
+
+1. **Job runner alerts**: The easiest place to start is with failure alerts
+from the [scheduled job
+runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) (cron,
+systemd, etc.) that's running borgmatic. But note that if the job doesn't even
+get scheduled (e.g. due to the job runner not running), you probably won't get
+an alert at all! Still, this is a decent first line of defense, especially
+when combined with some of the other approaches below.
+2. **borgmatic error hooks**: The `on_error` hook allows you to run an arbitrary
+command or script when borgmatic itself encounters an error running your
+backups. So for instance, you can run a script to send yourself a text message
+alert. But note that if borgmatic doesn't actually run, this alert won't fire.
+See [error
+hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
+below for how to configure this.
+4. **borgmatic Healthchecks hook**: This feature integrates with the
+[Healthchecks](https://healthchecks.io/) service, and pings Healthchecks
+whenever borgmatic runs. That way, Healthchecks can alert you when something
+goes wrong or it doesn't hear from borgmatic for a configured interval. See
+[Healthchecks
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
+below for how to configure this.
+3. **Third-party monitoring software**: You can use traditional monitoring
+software to consume borgmatic JSON output and track when the last
+successful backup occurred. See [scripting
+borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic)
+below for how to configure this.
+5. **Borg hosting providers**: Most [Borg hosting
+providers](https://torsion.org/borgmatic/#hosting-providers) include
+monitoring and alerting as part of their offering. This gives you a dashboard
+to check on all of your backups, and can alert you if the service doesn't hear
+from borgmatic for a configured interval.
+6. **borgmatic consistency checks**: While not strictly part of monitoring, if you
+really want confidence that your backups are not only running but are
+restorable as well, you can configure particular [consistency
+checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration)
+or even script full [restore
+tests](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/).
+
+
+## Error hooks
+
+When an error occurs during a backup, borgmatic can run configurable shell
+commands to fire off custom error notifications or take other actions, so you
+can get alerted as soon as something goes wrong. Here's a not-so-useful
+example:
+
+```yaml
+hooks:
+    on_error:
+        - echo "Error while creating a backup or running a backup hook."
+```
+
+The `on_error` hook supports interpolating particular runtime variables into
+the hook command. Here's an example that assumes you provide a separate shell
+script to handle the alerting:
+
+```yaml
+hooks:
+    on_error:
+        - send-text-message.sh "{configuration_filename}" "{repository}"
+```
+
+In this example, when the error occurs, borgmatic interpolates a few runtime
+values into the hook command: the borgmatic configuration filename, and the
+path of the repository. Here's the full set of supported variables you can use
+here:
+
+ * `configuration_filename`: borgmatic configuration filename in which the
+   error occurred
+ * `repository`: path of the repository in which the error occurred (may be
+   blank if the error occurs in a hook)
+ * `error`: the error message itself
+ * `output`: output of the command that failed (may be blank if an error
+   occurred without running a command)
+
+Note that borgmatic does not run `on_error` hooks if an error occurs within a
+`before_everything` or `after_everything` hook. For more about hooks, see the
+[borgmatic hooks
+documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/),
+especially the security information.
+
+
+## Healthchecks hook
+
+[Healthchecks](https://healthchecks.io/) is a service that provides "instant
+alerts when your cron jobs fail silently", and borgmatic has built-in
+integration with it. Once you create a Healthchecks account and project on
+their site, all you need to do is configure borgmatic with the unique "Ping
+URL" for your project. Here's an example:
+
+
+```yaml
+hooks:
+    healthchecks: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a
+```
+
+With this hook in place, borgmatic will ping your Healthchecks project when a
+backup begins, ends, or errors. Then you can configure Healthchecks to notify
+you by a [variety of
+mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail
+or it doesn't hear from borgmatic for a certain period of time.
+
+
+## Scripting borgmatic
+
+To consume the output of borgmatic in other software, you can include an
+optional `--json` flag with `create`, `list`, or `info` to get the output
+formatted as JSON.
+
+Note that when you specify the `--json` flag, Borg's other non-JSON output is
+suppressed so as not to interfere with the captured JSON. Also note that JSON
+output only shows up at the console, and not in syslog.
+
+
+### Successful backups
+
+`borgmatic list` includes support for a `--successful` flag that only lists
+successful (non-checkpoint) backups. This flag works via a basic heuristic: It
+assumes that non-checkpoint archive names end with a digit (e.g. from a
+timestamp), while checkpoint archive names do not. This means that if you're
+using custom archive names that do not end in a digit, the `--successful` flag
+will not work as expected.
+
+Combined with a built-in Borg flag like `--last`, you can list the last
+successful backup for use in your monitoring scripts. Here's an example
+combined with `--json`:
+
+```bash
+borgmatic list --successful --last 1 --json
+```
+
+Note that this particular combination will only work if you've got a single
+backup "series" in your repository. If you're instead backing up, say, from
+multiple different hosts into a single repository, then you'll need to get
+fancier with your archive listing. See `borg list --help` for more flags.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Restore a backup](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)

docs/how-to/restore-a-backup.md

@@ -63,5 +63,6 @@ Like a whole-archive restore, this also restores into the current directory.
 
 ## Related documentation
 
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
- * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)

docs/how-to/set-up-backups.md

@@ -77,7 +77,7 @@ else borgmatic won't recognize the option. Also be sure to use spaces rather
 than tabs for indentation; YAML does not allow tabs.
 
 You can also get the same sample configuration file from the [configuration
-reference](https://torsion.org/borgmatic/docs/reference/configuration.md), the authoritative set of
+reference](https://torsion.org/borgmatic/docs/reference/configuration/), the authoritative set of
 all configuration options. This is handy if borgmatic has added new options
 since you originally created your configuration file.
 
@@ -228,7 +228,7 @@ found character that cannot start any token
   in "config.yaml", line 230, column 1
 ```
 
-YAML does not allow tabs. So to fix this, simply replace any tabs in your
+YAML does not allow tabs. So to fix this, replace any tabs in your
 configuration file with the requisite number of spaces.
 
 ### libyaml compilation errors
@@ -244,13 +244,9 @@ it.
 
 ## Related documentation
 
- * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
- * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups.md)
- * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
- * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration.md)
- * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line.md)
-
-<script>
-  var links = document.getElementsByClassName("referral");
-  links[Math.floor(Math.random() * links.length)].style.display = "none";
-</script>
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+ * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)

docs/how-to/upgrade.md

@@ -76,4 +76,4 @@ files.
 
 ## Related documentation
 
- * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic.md)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)

docs/reference/command-line.md

@@ -13,5 +13,5 @@ each action sub-command:
 
 ## Related documentation
 
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
- * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)

docs/reference/configuration.md

@@ -15,5 +15,5 @@ file](https://torsion.org/borgmatic/docs/reference/config.yaml) for use locally.
 
 ## Related documentation
 
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
- * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)

scripts/dev-docs

@@ -2,9 +2,8 @@
 
 set -e
 
-docker build --tag borgmatic-docs --file docs/Dockerfile .
+docker build --tag borgmatic-docs --build-arg ENVIRONMENT=dev --file docs/Dockerfile .
 echo
 echo "You can view dev docs at http://localhost:8080"
-echo "Note that links within these docs will go to the online docs, so you will need to fiddle with URLs manually to stay in the dev docs."
 echo
 docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs

scripts/run-tests

@@ -7,7 +7,7 @@
 set -e
 
 python -m pip install --upgrade pip==19.1.1
-pip install tox==3.10.0
+pip install tox==3.14.0
 tox
 apk add --no-cache borgbackup
 tox -e end-to-end

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.3.24'
+VERSION = '1.4.0'
 
 
 setup(
@@ -31,6 +31,7 @@ setup(
     obsoletes=['atticmatic'],
     install_requires=(
         'pykwalify>=1.6.0,<14.06',
+        'requests',
         'ruamel.yaml>0.15.0,<0.17.0',
         'setuptools',
         'colorama>=0.4.1,<0.5',

test_requirements.txt

@@ -20,5 +20,6 @@ pytest==5.1.2
 pytest-cov==2.7.1
 python-dateutil==2.8.0
 PyYAML==5.1.2
+requests==2.22.0
 ruamel.yaml>0.15.0,<0.17.0
 toml==0.10.0

tests/integration/config/test_generate.py

@@ -40,6 +40,12 @@ def test_comment_out_line_comments_indented_option():
     assert module._comment_out_line(line) == '    # enabled: true'
 
 
+def test_comment_out_line_comments_twice_indented_option():
+    line = '        - item'
+
+    assert module._comment_out_line(line) == '        # - item'
+
+
 def test_comment_out_optional_configuration_comments_optional_config_only():
     flexmock(module)._comment_out_line = lambda line: '# ' + line
     config = '''
@@ -74,10 +80,10 @@ location:
     assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()
 
 
-def test_render_configuration_does_not_raise():
-    flexmock(module.yaml).should_receive('round_trip_dump')
+def test_render_configuration_converts_configuration_to_yaml_string():
+    yaml_string = module._render_configuration({'foo': 'bar'})
 
-    module._render_configuration({})
+    assert yaml_string == 'foo: bar\n'
 
 
 def test_write_configuration_does_not_raise():
@@ -107,12 +113,33 @@ def test_write_configuration_with_already_existing_directory_does_not_raise():
     module.write_configuration('config.yaml', 'config: yaml')
 
 
-def test_add_comments_to_configuration_does_not_raise():
+def test_add_comments_to_configuration_sequence_of_strings_does_not_raise():
+    config = module.yaml.comments.CommentedSeq(['foo', 'bar'])
+    schema = {'seq': [{'type': 'str'}]}
+
+    module.add_comments_to_configuration_sequence(config, schema)
+
+
+def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
+    config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
+    schema = {'seq': [{'map': {'foo': {'desc': 'yo'}}}]}
+
+    module.add_comments_to_configuration_sequence(config, schema)
+
+
+def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise():
+    config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
+    schema = {'seq': [{'map': {'foo': {}}}]}
+
+    module.add_comments_to_configuration_sequence(config, schema)
+
+
+def test_add_comments_to_configuration_map_does_not_raise():
     # Ensure that it can deal with fields both in the schema and missing from the schema.
     config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
     schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}}
 
-    module.add_comments_to_configuration(config, schema)
+    module.add_comments_to_configuration_map(config, schema)
 
 
 def test_generate_sample_configuration_does_not_raise():

tests/integration/test_execute.py

@@ -7,36 +7,57 @@ from flexmock import flexmock
 from borgmatic import execute as module
 
 
+def test_borg_command_identifies_borg_command():
+    assert module.borg_command(['/usr/bin/borg1', 'info'])
+
+
+def test_borg_command_does_not_identify_other_command():
+    assert not module.borg_command(['grep', 'stuff'])
+
+
 def test_execute_and_log_output_logs_each_line_separately():
     flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
     flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
+    flexmock(module).should_receive('borg_command').and_return(False)
 
-    module.execute_and_log_output(['echo', 'hi'], output_log_level=logging.INFO, shell=False)
-    module.execute_and_log_output(['echo', 'there'], output_log_level=logging.INFO, shell=False)
+    module.execute_and_log_output(
+        ['echo', 'hi'], output_log_level=logging.INFO, shell=False, environment=None
+    )
+    module.execute_and_log_output(
+        ['echo', 'there'], output_log_level=logging.INFO, shell=False, environment=None
+    )
 
 
 def test_execute_and_log_output_with_borg_warning_does_not_raise():
     flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('borg_command').and_return(True)
 
-    # Borg's exit code 1 is a warning, not an error.
-    module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False)
+    module.execute_and_log_output(
+        ['false'], output_log_level=logging.INFO, shell=False, environment=None
+    )
 
 
 def test_execute_and_log_output_includes_borg_error_output_in_exception():
     flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('borg_command').and_return(True)
 
     with pytest.raises(subprocess.CalledProcessError) as error:
-        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
+        module.execute_and_log_output(
+            ['grep'], output_log_level=logging.INFO, shell=False, environment=None
+        )
 
     assert error.value.returncode == 2
     assert error.value.output
 
 
-def test_execute_and_log_output_with_shell_error_raises():
+def test_execute_and_log_output_with_non_borg_error_raises():
     flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('borg_command').and_return(False)
 
     with pytest.raises(subprocess.CalledProcessError) as error:
-        module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=True)
+        module.execute_and_log_output(
+            ['false'], output_log_level=logging.INFO, shell=False, environment=None
+        )
 
     assert error.value.returncode == 1
 
@@ -44,9 +65,12 @@ def test_execute_and_log_output_with_shell_error_raises():
 def test_execute_and_log_output_truncates_long_borg_error_output():
     flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
     flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('borg_command').and_return(False)
 
     with pytest.raises(subprocess.CalledProcessError) as error:
-        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
+        module.execute_and_log_output(
+            ['grep'], output_log_level=logging.INFO, shell=False, environment=None
+        )
 
     assert error.value.returncode == 2
     assert error.value.output.startswith('...')
@@ -54,12 +78,18 @@ def test_execute_and_log_output_truncates_long_borg_error_output():
 
 def test_execute_and_log_output_with_no_output_logs_nothing():
     flexmock(module.logger).should_receive('log').never()
+    flexmock(module).should_receive('borg_command').and_return(False)
 
-    module.execute_and_log_output(['true'], output_log_level=logging.INFO, shell=False)
+    module.execute_and_log_output(
+        ['true'], output_log_level=logging.INFO, shell=False, environment=None
+    )
 
 
 def test_execute_and_log_output_with_error_exit_status_raises():
     flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('borg_command').and_return(False)
 
     with pytest.raises(subprocess.CalledProcessError):
-        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
+        module.execute_and_log_output(
+            ['grep'], output_log_level=logging.INFO, shell=False, environment=None
+        )

tests/unit/borg/test_create.py

@@ -156,11 +156,26 @@ def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
     assert exclude_flags == ()
 
 
+def test_borgmatic_source_directories_set_when_directory_exists():
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.os.path).should_receive('expanduser')
+
+    assert module.borgmatic_source_directories() == [module.BORGMATIC_SOURCE_DIRECTORY]
+
+
+def test_borgmatic_source_directories_empty_when_directory_does_not_exist():
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os.path).should_receive('expanduser')
+
+    assert module.borgmatic_source_directories() == []
+
+
 DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
 ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 
 
 def test_create_archive_calls_borg_with_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -184,6 +199,7 @@ def test_create_archive_calls_borg_with_parameters():
 
 def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(
@@ -209,6 +225,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
@@ -233,6 +250,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 
 def test_create_archive_with_log_info_calls_borg_with_info_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -258,6 +276,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
 
 
 def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -283,6 +302,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
 
 
 def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -308,6 +328,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
 
 
 def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -332,6 +353,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
 
 
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -357,6 +379,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
 def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_parameter():
     # --dry-run and --stats are mutually exclusive, see:
     # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -385,6 +408,7 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param
 def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_parameter():
     # --dry-run and --stats are mutually exclusive, see:
     # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -411,6 +435,7 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para
 
 
 def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -434,6 +459,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
 
 
 def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -457,6 +483,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
 
 
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -480,6 +507,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -503,6 +531,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 
 
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -526,6 +555,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 
 
 def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -549,6 +579,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
 
 
 def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -573,6 +604,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
 
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -597,6 +629,7 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
 
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -621,6 +654,7 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
 
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -645,6 +679,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 
 def test_create_archive_with_local_path_calls_borg_via_local_path():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -668,6 +703,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -692,6 +728,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -714,6 +751,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
 
 def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -736,6 +774,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
 
 
 def test_create_archive_with_stats_calls_borg_with_stats_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -759,6 +798,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter():
 
 
 def test_create_archive_with_progress_calls_borg_with_progress_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -782,6 +822,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
 
 
 def test_create_archive_with_json_calls_borg_with_json_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -807,6 +848,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
 
 
 def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -833,6 +875,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
 
 
 def test_create_archive_with_source_directories_glob_expands():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -857,6 +900,7 @@ def test_create_archive_with_source_directories_glob_expands():
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -881,6 +925,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -904,6 +949,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -926,6 +972,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)

tests/unit/borg/test_list.py

@@ -147,7 +147,7 @@ def test_list_archives_with_short_calls_borg_with_short_parameter():
         'exclude',
         'exclude_from',
         'pattern',
-        'pattern_from',
+        'patterns_from',
     ),
 )
 def test_list_archives_passes_through_arguments_to_borg(argument_name):

tests/unit/commands/test_borgmatic.py

@@ -22,7 +22,10 @@ def test_run_configuration_runs_actions_for_each_repository():
 
 def test_run_configuration_executes_hooks_for_create_action():
     flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.hook).should_receive('execute_hook').twice()
+    flexmock(module.command).should_receive('execute_hook').twice()
+    flexmock(module.postgresql).should_receive('dump_databases').once()
+    flexmock(module.healthchecks).should_receive('ping_healthchecks').twice()
+    flexmock(module.postgresql).should_receive('remove_database_dumps').once()
     flexmock(module).should_receive('run_actions').and_return([])
     config = {'location': {'repositories': ['foo']}}
     arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
@@ -32,7 +35,9 @@ def test_run_configuration_executes_hooks_for_create_action():
 
 def test_run_configuration_logs_actions_error():
     flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.hook).should_receive('execute_hook')
+    flexmock(module.command).should_receive('execute_hook')
+    flexmock(module.postgresql).should_receive('dump_databases')
+    flexmock(module.healthchecks).should_receive('ping_healthchecks')
     expected_results = [flexmock()]
     flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
     flexmock(module).should_receive('run_actions').and_raise(OSError)
@@ -46,7 +51,7 @@ def test_run_configuration_logs_actions_error():
 
 def test_run_configuration_logs_pre_hook_error():
     flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.hook).should_receive('execute_hook').and_raise(OSError).and_return(None)
+    flexmock(module.command).should_receive('execute_hook').and_raise(OSError).and_return(None)
     expected_results = [flexmock()]
     flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
     flexmock(module).should_receive('run_actions').never()
@@ -60,7 +65,7 @@ def test_run_configuration_logs_pre_hook_error():
 
 def test_run_configuration_logs_post_hook_error():
     flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.hook).should_receive('execute_hook').and_return(None).and_raise(
+    flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
         OSError
     ).and_return(None)
     expected_results = [flexmock()]
@@ -76,7 +81,7 @@ def test_run_configuration_logs_post_hook_error():
 
 def test_run_configuration_logs_on_error_hook_error():
     flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.hook).should_receive('execute_hook').and_raise(OSError)
+    flexmock(module.command).should_receive('execute_hook').and_raise(OSError)
     expected_results = [flexmock(), flexmock()]
     flexmock(module).should_receive('make_error_log_records').and_return(
         expected_results[:1]
@@ -148,7 +153,7 @@ def test_make_error_log_records_generates_nothing_for_other_error():
 
 
 def test_collect_configuration_run_summary_logs_info_for_success():
-    flexmock(module.hook).should_receive('execute_hook').never()
+    flexmock(module.command).should_receive('execute_hook').never()
     flexmock(module).should_receive('run_configuration').and_return([])
     arguments = {}
 
@@ -208,7 +213,7 @@ def test_collect_configuration_run_summary_logs_missing_configs_error():
 
 
 def test_collect_configuration_run_summary_logs_pre_hook_error():
-    flexmock(module.hook).should_receive('execute_hook').and_raise(ValueError)
+    flexmock(module.command).should_receive('execute_hook').and_raise(ValueError)
     expected_logs = (flexmock(),)
     flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
     arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
@@ -221,7 +226,7 @@ def test_collect_configuration_run_summary_logs_pre_hook_error():
 
 
 def test_collect_configuration_run_summary_logs_post_hook_error():
-    flexmock(module.hook).should_receive('execute_hook').and_return(None).and_raise(ValueError)
+    flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(ValueError)
     flexmock(module).should_receive('run_configuration').and_return([])
     expected_logs = (flexmock(),)
     flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)

tests/unit/config/test_generate.py

@@ -1,13 +1,14 @@
 from collections import OrderedDict
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.config import generate as module
 
 
-def test_schema_to_sample_configuration_generates_config_with_examples():
+def test_schema_to_sample_configuration_generates_config_map_with_examples():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
-    flexmock(module).should_receive('add_comments_to_configuration')
+    flexmock(module).should_receive('add_comments_to_configuration_map')
     schema = {
         'map': OrderedDict(
             [
@@ -35,3 +36,38 @@ def test_schema_to_sample_configuration_generates_config_with_examples():
             ('section2', OrderedDict([('field2', 'Example 2'), ('field3', 'Example 3')])),
         ]
     )
+
+
+def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example():
+    flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
+    flexmock(module).should_receive('add_comments_to_configuration_sequence')
+    schema = {'seq': [{'type': 'str'}], 'example': ['hi']}
+
+    config = module._schema_to_sample_configuration(schema)
+
+    assert config == ['hi']
+
+
+def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_examples():
+    flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
+    flexmock(module).should_receive('add_comments_to_configuration_sequence')
+    schema = {
+        'seq': [
+            {
+                'map': OrderedDict(
+                    [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})]
+                )
+            }
+        ]
+    }
+
+    config = module._schema_to_sample_configuration(schema)
+
+    assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])]
+
+
+def test_schema_to_sample_configuration_with_unsupported_schema_raises():
+    schema = {'gobbledygook': [{'type': 'not-your'}]}
+
+    with pytest.raises(ValueError):
+        module._schema_to_sample_configuration(schema)

tests/unit/config/test_validate.py

@@ -74,6 +74,27 @@ def test_apply_logical_validation_does_not_raise_otherwise():
     module.apply_logical_validation('config.yaml', {'retention': {'keep_secondly': 1000}})
 
 
+def test_remove_examples_strips_examples_from_map():
+    schema = {
+        'map': {
+            'foo': {'desc': 'thing1', 'example': 'bar'},
+            'baz': {'desc': 'thing2', 'example': 'quux'},
+        }
+    }
+
+    module.remove_examples(schema)
+
+    assert schema == {'map': {'foo': {'desc': 'thing1'}, 'baz': {'desc': 'thing2'}}}
+
+
+def test_remove_examples_strips_examples_from_sequence_of_maps():
+    schema = {'seq': [{'map': {'foo': {'desc': 'thing', 'example': 'bar'}}, 'example': 'stuff'}]}
+
+    module.remove_examples(schema)
+
+    assert schema == {'seq': [{'map': {'foo': {'desc': 'thing'}}}]}
+
+
 def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config():
     module.guard_configuration_contains_repository(
         repository='repo', configurations={'config.yaml': {'location': {'repositories': ['repo']}}}

tests/unit/hooks/test_command.py

@@ -2,7 +2,7 @@ import logging
 
 from flexmock import flexmock
 
-from borgmatic import hook as module
+from borgmatic.hooks import command as module
 
 
 def test_interpolate_context_passes_through_command_without_variable():

tests/unit/hooks/test_healthchecks.py

@@ -0,0 +1,33 @@
+from flexmock import flexmock
+
+from borgmatic.hooks import healthchecks as module
+
+
+def test_ping_healthchecks_hits_ping_url():
+    ping_url = 'https://example.com'
+    flexmock(module.requests).should_receive('get').with_args(ping_url)
+
+    module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False)
+
+
+def test_ping_healthchecks_without_ping_url_does_not_raise():
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_healthchecks(ping_url_or_uuid=None, config_filename='config.yaml', dry_run=False)
+
+
+def test_ping_healthchecks_with_ping_uuid_hits_corresponding_url():
+    ping_uuid = 'abcd-efgh-ijkl-mnop'
+    flexmock(module.requests).should_receive('get').with_args(
+        'https://hc-ping.com/{}'.format(ping_uuid)
+    )
+
+    module.ping_healthchecks(ping_uuid, 'config.yaml', dry_run=False)
+
+
+def test_ping_healthchecks_hits_ping_url_with_append():
+    ping_url = 'https://example.com'
+    append = 'failed-so-hard'
+    flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
+
+    module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False, append=append)

tests/unit/hooks/test_postgresql.py

@@ -0,0 +1,187 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.hooks import postgresql as module
+
+
+def test_dump_databases_runs_pg_dump_for_each_database():
+    databases = [{'name': 'foo'}, {'name': 'bar'}]
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases')
+    flexmock(module.os).should_receive('makedirs')
+
+    for name in ('foo', 'bar'):
+        flexmock(module).should_receive('execute_command').with_args(
+            (
+                'pg_dump',
+                '--no-password',
+                '--clean',
+                '--file',
+                'databases/localhost/{}'.format(name),
+                '--format',
+                'custom',
+                name,
+            ),
+            extra_environment=None,
+        ).once()
+
+    module.dump_databases(databases, 'test.yaml', dry_run=False)
+
+
+def test_dump_databases_with_dry_run_skips_pg_dump():
+    databases = [{'name': 'foo'}, {'name': 'bar'}]
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases')
+    flexmock(module.os).should_receive('makedirs')
+    flexmock(module).should_receive('execute_command').never()
+
+    module.dump_databases(databases, 'test.yaml', dry_run=True)
+
+
+def test_dump_databases_without_databases_does_not_raise():
+    module.dump_databases([], 'test.yaml', dry_run=False)
+
+
+def test_dump_databases_with_invalid_database_name_raises():
+    databases = [{'name': 'heehee/../../etc/passwd'}]
+
+    with pytest.raises(ValueError):
+        module.dump_databases(databases, 'test.yaml', dry_run=True)
+
+
+def test_dump_databases_runs_pg_dump_with_hostname_and_port():
+    databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases')
+    flexmock(module.os).should_receive('makedirs')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'pg_dump',
+            '--no-password',
+            '--clean',
+            '--file',
+            'databases/database.example.org/foo',
+            '--host',
+            'database.example.org',
+            '--port',
+            '5433',
+            '--format',
+            'custom',
+            'foo',
+        ),
+        extra_environment=None,
+    ).once()
+
+    module.dump_databases(databases, 'test.yaml', dry_run=False)
+
+
+def test_dump_databases_runs_pg_dump_with_username_and_password():
+    databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases')
+    flexmock(module.os).should_receive('makedirs')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'pg_dump',
+            '--no-password',
+            '--clean',
+            '--file',
+            'databases/localhost/foo',
+            '--username',
+            'postgres',
+            '--format',
+            'custom',
+            'foo',
+        ),
+        extra_environment={'PGPASSWORD': 'trustsome1'},
+    ).once()
+
+    module.dump_databases(databases, 'test.yaml', dry_run=False)
+
+
+def test_dump_databases_runs_pg_dump_with_format():
+    databases = [{'name': 'foo', 'format': 'tar'}]
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases')
+    flexmock(module.os).should_receive('makedirs')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'pg_dump',
+            '--no-password',
+            '--clean',
+            '--file',
+            'databases/localhost/foo',
+            '--format',
+            'tar',
+            'foo',
+        ),
+        extra_environment=None,
+    ).once()
+
+    module.dump_databases(databases, 'test.yaml', dry_run=False)
+
+
+def test_dump_databases_runs_pg_dump_with_options():
+    databases = [{'name': 'foo', 'options': '--stuff=such'}]
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases')
+    flexmock(module.os).should_receive('makedirs')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'pg_dump',
+            '--no-password',
+            '--clean',
+            '--file',
+            'databases/localhost/foo',
+            '--format',
+            'custom',
+            '--stuff=such',
+            'foo',
+        ),
+        extra_environment=None,
+    ).once()
+
+    module.dump_databases(databases, 'test.yaml', dry_run=False)
+
+
+def test_dump_databases_runs_pg_dumpall_for_all_databases():
+    databases = [{'name': 'all'}]
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases')
+    flexmock(module.os).should_receive('makedirs')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        ('pg_dumpall', '--no-password', '--clean', '--file', 'databases/localhost/all'),
+        extra_environment=None,
+    ).once()
+
+    module.dump_databases(databases, 'test.yaml', dry_run=False)
+
+
+def test_remove_database_dumps_removes_dump_for_each_database():
+    databases = [{'name': 'foo'}, {'name': 'bar'}]
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases')
+    flexmock(module.os).should_receive('listdir').and_return([])
+    flexmock(module.os).should_receive('rmdir')
+
+    for name in ('foo', 'bar'):
+        flexmock(module.os).should_receive('remove').with_args(
+            'databases/localhost/{}'.format(name)
+        ).once()
+
+    module.remove_database_dumps(databases, 'test.yaml', dry_run=False)
+
+
+def test_remove_database_dumps_with_dry_run_skips_removal():
+    databases = [{'name': 'foo'}, {'name': 'bar'}]
+    flexmock(module.os).should_receive('remove').never()
+
+    module.remove_database_dumps(databases, 'test.yaml', dry_run=True)
+
+
+def test_remove_database_dumps_without_databases_does_not_raise():
+    module.remove_database_dumps([], 'test.yaml', dry_run=False)
+
+
+def test_remove_database_dumps_with_invalid_database_name_raises():
+    databases = [{'name': 'heehee/../../etc/passwd'}]
+
+    with pytest.raises(ValueError):
+        module.remove_database_dumps(databases, 'test.yaml', dry_run=True)

tests/unit/test_execute.py

@@ -8,8 +8,9 @@ from borgmatic import execute as module
 
 def test_execute_command_calls_full_command():
     full_command = ['foo', 'bar']
+    flexmock(module.os, environ={'a': 'b'})
     flexmock(module).should_receive('execute_and_log_output').with_args(
-        full_command, output_log_level=logging.INFO, shell=False
+        full_command, output_log_level=logging.INFO, shell=False, environment=None
     ).once()
 
     output = module.execute_command(full_command)
@@ -19,8 +20,9 @@ def test_execute_command_calls_full_command():
 
 def test_execute_command_calls_full_command_with_shell():
     full_command = ['foo', 'bar']
+    flexmock(module.os, environ={'a': 'b'})
     flexmock(module).should_receive('execute_and_log_output').with_args(
-        full_command, output_log_level=logging.INFO, shell=True
+        full_command, output_log_level=logging.INFO, shell=True, environment=None
     ).once()
 
     output = module.execute_command(full_command, shell=True)
@@ -28,11 +30,24 @@ def test_execute_command_calls_full_command_with_shell():
     assert output is None
 
 
+def test_execute_command_calls_full_command_with_extra_environment():
+    full_command = ['foo', 'bar']
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module).should_receive('execute_and_log_output').with_args(
+        full_command, output_log_level=logging.INFO, shell=False, environment={'a': 'b', 'c': 'd'}
+    ).once()
+
+    output = module.execute_command(full_command, extra_environment={'c': 'd'})
+
+    assert output is None
+
+
 def test_execute_command_captures_output():
     full_command = ['foo', 'bar']
     expected_output = '[]'
+    flexmock(module.os, environ={'a': 'b'})
     flexmock(module.subprocess).should_receive('check_output').with_args(
-        full_command, shell=False
+        full_command, shell=False, env=None
     ).and_return(flexmock(decode=lambda: expected_output)).once()
 
     output = module.execute_command(full_command, output_log_level=None)
@@ -43,8 +58,9 @@ def test_execute_command_captures_output():
 def test_execute_command_captures_output_with_shell():
     full_command = ['foo', 'bar']
     expected_output = '[]'
+    flexmock(module.os, environ={'a': 'b'})
     flexmock(module.subprocess).should_receive('check_output').with_args(
-        full_command, shell=True
+        full_command, shell=True, env=None
     ).and_return(flexmock(decode=lambda: expected_output)).once()
 
     output = module.execute_command(full_command, output_log_level=None, shell=True)
@@ -52,6 +68,21 @@ def test_execute_command_captures_output_with_shell():
     assert output == expected_output
 
 
+def test_execute_command_captures_output_with_extra_environment():
+    full_command = ['foo', 'bar']
+    expected_output = '[]'
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('check_output').with_args(
+        full_command, shell=False, env={'a': 'b', 'c': 'd'}
+    ).and_return(flexmock(decode=lambda: expected_output)).once()
+
+    output = module.execute_command(
+        full_command, output_log_level=None, shell=False, extra_environment={'c': 'd'}
+    )
+
+    assert output == expected_output
+
+
 def test_execute_command_without_capture_does_not_raise_on_success():
     flexmock(module.subprocess).should_receive('check_call').and_raise(
         module.subprocess.CalledProcessError(0, 'borg init')

tox.ini

@@ -2,7 +2,7 @@
 envlist = py35,py36,py37
 skip_missing_interpreters = True
 skipsdist = True
-minversion = 3.10.0
+minversion = 3.14.0
 
 [testenv]
 usedevelop = True