Backup to a removable drive or intermittent server via "soft failure" feature (#284).

This commit is contained in:
Dan Helfman 2020-01-24 20:52:48 -08:00
parent fdbb2ee905
commit 2405e97c38
8 changed files with 212 additions and 3 deletions

3
NEWS
View File

@ -3,6 +3,9 @@
* #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag.
* #280: Change "exclude_if_present" option to support multiple filenames that indicate a directory
should be excluded from backups, rather than just a single filename.
* #284: Backup to a removable drive or intermittent server via "soft failure" feature. See the
documentation for more information:
https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/
* #287: View consistency check progress via "--progress" flag for "check" action.
* For "create" and "prune" actions, no longer list files or show detailed stats at any verbosities
by default. You can opt back in with "--files" or "--stats" flags.

View File

@ -24,7 +24,7 @@ location:
repositories:
- 1234@usw-s001.rsync.net:backups.borg
- k8pDxu32@k8pDxu32.repo.borgbase.com:repo
- /var/lib/backups/backups.borg
- /var/lib/backups/local.borg
retention:
# Retention policy for how many backups to keep.
@ -80,6 +80,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
* [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/)
* [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
* [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
* [Backup to a removable drive or an intermittent server](https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/)
* [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
* [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)

View File

@ -83,6 +83,9 @@ def run_configuration(config_filename, config, arguments):
global_arguments.dry_run,
)
except (OSError, CalledProcessError) as error:
if command.considered_soft_failure(config_filename, error):
return
encountered_error = error
yield from make_error_log_records(
'{}: Error running pre-backup hook'.format(config_filename), error
@ -138,6 +141,9 @@ def run_configuration(config_filename, config, arguments):
global_arguments.dry_run,
)
except (OSError, CalledProcessError) as error:
if command.considered_soft_failure(config_filename, error):
return
encountered_error = error
yield from make_error_log_records(
'{}: Error running post-backup hook'.format(config_filename), error
@ -165,6 +171,9 @@ def run_configuration(config_filename, config, arguments):
global_arguments.dry_run,
)
except (OSError, CalledProcessError) as error:
if command.considered_soft_failure(config_filename, error):
return
yield from make_error_log_records(
'{}: Error running on-error hook'.format(config_filename), error
)

View File

@ -548,7 +548,8 @@ map:
- type: str
desc: |
List of one or more shell commands or scripts to execute before running all
actions (if one of them is "create"), run once before all configuration files.
actions (if one of them is "create"). These are collected from all configuration
files and then run once before all of them (prior to all actions).
example:
- echo "Starting actions."
after_everything:
@ -556,7 +557,8 @@ map:
- type: str
desc: |
List of one or more shell commands or scripts to execute after running all
actions (if one of them is "create"), run once after all configuration files.
actions (if one of them is "create"). These are collected from all configuration
files and then run once before all of them (prior to all actions).
example:
- echo "Completed actions."
umask:

View File

@ -6,6 +6,9 @@ from borgmatic import execute
logger = logging.getLogger(__name__)
SOFT_FAIL_EXIT_CODE = 75
def interpolate_context(command, context):
'''
Given a single hook command and a dict of context names/values, interpolate the values by
@ -69,3 +72,24 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
finally:
if original_umask:
os.umask(original_umask)
def considered_soft_failure(config_filename, error):
'''
Given a configuration filename and an exception object, return whether the exception object
represents a subprocess.CalledProcessError with a return code of SOFT_FAIL_EXIT_CODE. If so,
that indicates that the error is a "soft failure", and should not result in an error.
'''
exit_code = getattr(error, 'returncode', None)
if exit_code is None:
return False
if exit_code == SOFT_FAIL_EXIT_CODE:
logger.info(
'{}: Command hook exited with soft failure exit code ({}); skipping remaining actions'.format(
config_filename, SOFT_FAIL_EXIT_CODE
)
)
return True
return False

View File

@ -0,0 +1,106 @@
---
title: Backup to a removable drive or an intermittent server
---
## Occasional backups
A common situation is backing up to a repository that's only sometimes online.
For instance, you might send most of your backups to the cloud, but
occasionally you want to plug in an external hard drive or backup to your
buddy's sometimes-online server for that extra level of redundancy.
But if you run borgmatic and your hard drive isn't plugged in, or your buddy's
server is offline, then you'll get an annoying error message and the overall
borgmatic run will fail (even if individual repositories complete just fine).
So what if you want borgmatic to swallow the error of a missing drive
or an offline server, and continue trucking along? That's where the concept of
"soft failure" come in.
## Soft failure command hooks
This feature leverages [borgmatic command
hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/),
so first familiarize yourself with them. The idea is that you write a simple
test in the form of a borgmatic hook to see if backups should proceed or not.
The way the test works is that if any of your hook commands return a special
exit status of 75, that indicates to borgmatic that it's a temporary failure,
and borgmatic should skip all subsequent actions for that configuration file.
If you return any other status, then it's a standard success or error. (Zero is
success; anything else other than 75 is an error).
So for instance, if you have an external drive that's only sometimes mounted,
declare its repository in its own [separate configuration
file](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/),
say at `/etc/borgmatic.d/removable.yaml`:
```yaml
location:
source_directories:
- /home
repositories:
- /mnt/removable/backup.borg
```
Then, write a `before_backup` hook in that same configuration file that uses
the external `findmnt` utility to see whether the drive is mounted before
proceeding.
```yaml
hooks:
before_backup:
- findmnt /mnt/removable > /dev/null || exit 75
```
What this does is check if the `findmnt` command errors when probing for a
particular mount point. If it does error, then it returns exit code 75 to
borgmatic. borgmatic logs the soft failure, skips all further actions in that
configurable file, and proceeds onward to any other borgmatic configuration
files you may have.
You can imagine a similar check for the sometimes-online server case:
```yaml
location:
source_directories:
- /home
repositories:
- me@buddys-server.org:backup.borg
hooks:
before_backup:
- ping -q -c 1 buddys-server.org > /dev/null || exit 75
```
## Caveats and details
There are some caveats you should be aware of with this feature.
* You'll generally want to put a soft failure command in the `before_backup`
hook, so as to gate whether the backup action occurs. While a soft failure is
also supported in the `after_backup` hook, returning a soft failure there
won't prevent any actions from occuring, because they've already occurred!
Similiarly, you can return a soft failure from an `on_error` hook, but at
that point it's too late to prevent the error.
* Returning a soft failure does prevent further commands in the same hook from
executing. So, like a standard error, it is an "early out". Unlike a standard
error, borgmatic does not display it in angry red text or consider it a
failure.
* The soft failure only applies to the scope of a single borgmatic
configuration file. So put anything that you don't want soft-failed, like
always-online cloud backups, in separate configuration files from your
soft-failing repositories.
* The soft failure doesn't have to apply to a repository. You can even perform
a test to make sure that individual source directories are mounted and
available. Use your imagination!
* This feature does not apply to `before_everything` or `after_everything`
hooks.
## Related documentation
* [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
* [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
* [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
* [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)

View File

@ -3,6 +3,7 @@ import subprocess
from flexmock import flexmock
import borgmatic.hooks.command
from borgmatic.commands import borgmatic as module
@ -93,6 +94,20 @@ def test_run_configuration_logs_pre_hook_error():
assert results == expected_results
def test_run_configuration_bails_for_pre_hook_soft_failure():
flexmock(module.borg_environment).should_receive('initialize')
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
flexmock(module.command).should_receive('execute_hook').and_raise(error).and_return(None)
flexmock(module).should_receive('make_error_log_records').never()
flexmock(module).should_receive('run_actions').never()
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == []
def test_run_configuration_logs_post_hook_error():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
@ -110,6 +125,23 @@ def test_run_configuration_logs_post_hook_error():
assert results == expected_results
def test_run_configuration_bails_for_post_hook_soft_failure():
flexmock(module.borg_environment).should_receive('initialize')
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
error
).and_return(None)
flexmock(module.dispatch).should_receive('call_hooks')
flexmock(module).should_receive('make_error_log_records').never()
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == []
def test_run_configuration_logs_on_error_hook_error():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook').and_raise(OSError)
@ -126,6 +158,21 @@ def test_run_configuration_logs_on_error_hook_error():
assert results == expected_results
def test_run_configuration_bails_for_on_error_hook_soft_failure():
flexmock(module.borg_environment).should_receive('initialize')
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(error)
expected_results = [flexmock()]
flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_raise(OSError)
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_load_configurations_collects_parsed_configurations():
configuration = flexmock()
other_configuration = flexmock()

View File

@ -1,4 +1,5 @@
import logging
import subprocess
from flexmock import flexmock
@ -79,3 +80,19 @@ def test_execute_hook_on_error_logs_as_error():
).once()
module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False)
def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail():
error = subprocess.CalledProcessError(module.SOFT_FAIL_EXIT_CODE, 'try again')
assert module.considered_soft_failure('config.yaml', error)
def test_considered_soft_failure_does_not_treat_other_exit_code_as_soft_fail():
error = subprocess.CalledProcessError(1, 'error')
assert not module.considered_soft_failure('config.yaml', error)
def test_considered_soft_failure_does_not_treat_other_exception_type_as_soft_fail():
assert not module.considered_soft_failure('config.yaml', Exception())