Instead of executing "before" command hooks before all borgmatic actions run (and "after" hooks after), execute these hooks right before/after the corresponding action (#473).
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Dan Helfman 2022-04-21 22:08:25 -07:00
parent cbce6707f4
commit ed7fe5c6d0
5 changed files with 478 additions and 220 deletions

9
NEWS
View File

@ -1,4 +1,11 @@
1.5.25.dev0 1.6.0.dev0
* #473: Instead of executing "before" command hooks before all borgmatic actions run (and "after"
hooks after), execute these hooks right before/after the corresponding action. E.g.,
"before_check" now runs immediately before the "check" action. This better supports running
timing-sensitive tasks like pausing containers. Side effect: before/after command hooks now run
once for each configured repository instead of once per configuration file. Additionally, the
"repositories" interpolated variable has been changed to "repository", containing the path to the
current repository for the hook.
* #516: Fix handling of TERM signal to exit borgmatic, not just forward the signal to Borg. * #516: Fix handling of TERM signal to exit borgmatic, not just forward the signal to Borg.
* #517: Fix borgmatic exit code (so it's zero) when initial Borg calls fail but later retries * #517: Fix borgmatic exit code (so it's zero) when initial Borg calls fail but later retries
succeed. succeed.

View File

@ -65,10 +65,6 @@ def run_configuration(config_filename, config, arguments):
using_primary_action = {'prune', 'compact', 'create', 'check'}.intersection(arguments) using_primary_action = {'prune', 'compact', 'create', 'check'}.intersection(arguments)
monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity)
hook_context = {
'repositories': ','.join(location['repositories']),
}
try: try:
local_borg_version = borg_version.local_borg_version(local_path) local_borg_version = borg_version.local_borg_version(local_path)
except (OSError, CalledProcessError, ValueError) as error: except (OSError, CalledProcessError, ValueError) as error:
@ -87,50 +83,6 @@ def run_configuration(config_filename, config, arguments):
monitoring_log_level, monitoring_log_level,
global_arguments.dry_run, global_arguments.dry_run,
) )
if 'prune' in arguments:
command.execute_hook(
hooks.get('before_prune'),
hooks.get('umask'),
config_filename,
'pre-prune',
global_arguments.dry_run,
**hook_context,
)
if 'compact' in arguments:
command.execute_hook(
hooks.get('before_compact'),
hooks.get('umask'),
config_filename,
'pre-compact',
global_arguments.dry_run,
)
if 'create' in arguments:
command.execute_hook(
hooks.get('before_backup'),
hooks.get('umask'),
config_filename,
'pre-backup',
global_arguments.dry_run,
**hook_context,
)
if 'check' in arguments:
command.execute_hook(
hooks.get('before_check'),
hooks.get('umask'),
config_filename,
'pre-check',
global_arguments.dry_run,
**hook_context,
)
if 'extract' in arguments:
command.execute_hook(
hooks.get('before_extract'),
hooks.get('umask'),
config_filename,
'pre-extract',
global_arguments.dry_run,
**hook_context,
)
if using_primary_action: if using_primary_action:
dispatch.call_hooks( dispatch.call_hooks(
'ping_monitor', 'ping_monitor',
@ -146,7 +98,7 @@ def run_configuration(config_filename, config, arguments):
return return
encountered_error = error encountered_error = error
yield from log_error_records('{}: Error running pre hook'.format(config_filename), error) yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error)
if not encountered_error: if not encountered_error:
repo_queue = Queue() repo_queue = Queue()
@ -162,6 +114,7 @@ def run_configuration(config_filename, config, arguments):
try: try:
yield from run_actions( yield from run_actions(
arguments=arguments, arguments=arguments,
config_filename=config_filename,
location=location, location=location,
storage=storage, storage=storage,
retention=retention, retention=retention,
@ -188,6 +141,9 @@ def run_configuration(config_filename, config, arguments):
) )
continue continue
if command.considered_soft_failure(config_filename, error):
return
yield from log_error_records( yield from log_error_records(
'{}: Error running actions for repository'.format(repository_path), error '{}: Error running actions for repository'.format(repository_path), error
) )
@ -196,58 +152,6 @@ def run_configuration(config_filename, config, arguments):
if not encountered_error: if not encountered_error:
try: try:
if 'prune' in arguments:
command.execute_hook(
hooks.get('after_prune'),
hooks.get('umask'),
config_filename,
'post-prune',
global_arguments.dry_run,
**hook_context,
)
if 'compact' in arguments:
command.execute_hook(
hooks.get('after_compact'),
hooks.get('umask'),
config_filename,
'post-compact',
global_arguments.dry_run,
)
if 'create' in arguments:
dispatch.call_hooks(
'remove_database_dumps',
hooks,
config_filename,
dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
command.execute_hook(
hooks.get('after_backup'),
hooks.get('umask'),
config_filename,
'post-backup',
global_arguments.dry_run,
**hook_context,
)
if 'check' in arguments:
command.execute_hook(
hooks.get('after_check'),
hooks.get('umask'),
config_filename,
'post-check',
global_arguments.dry_run,
**hook_context,
)
if 'extract' in arguments:
command.execute_hook(
hooks.get('after_extract'),
hooks.get('umask'),
config_filename,
'post-extract',
global_arguments.dry_run,
**hook_context,
)
if using_primary_action: if using_primary_action:
dispatch.call_hooks( dispatch.call_hooks(
'ping_monitor', 'ping_monitor',
@ -271,9 +175,7 @@ def run_configuration(config_filename, config, arguments):
return return
encountered_error = error encountered_error = error
yield from log_error_records( yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error)
'{}: Error running post hook'.format(config_filename), error
)
if encountered_error and using_primary_action: if encountered_error and using_primary_action:
try: try:
@ -316,6 +218,7 @@ def run_configuration(config_filename, config, arguments):
def run_actions( def run_actions(
*, *,
arguments, arguments,
config_filename,
location, location,
storage, storage,
retention, retention,
@ -325,20 +228,28 @@ def run_actions(
remote_path, remote_path,
local_borg_version, local_borg_version,
repository_path, repository_path,
): # pragma: no cover ):
''' '''
Given parsed command-line arguments as an argparse.ArgumentParser instance, several different Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration
configuration dicts, local and remote paths to Borg, a local Borg version string, and a filename, several different configuration dicts, local and remote paths to Borg, a local Borg
repository name, run all actions from the command-line arguments on the given repository. version string, and a repository name, run all actions from the command-line arguments on the
given repository.
Yield JSON output strings from executing any actions that produce JSON. Yield JSON output strings from executing any actions that produce JSON.
Raise OSError or subprocess.CalledProcessError if an error occurs running a command for an Raise OSError or subprocess.CalledProcessError if an error occurs running a command for an
action. Raise ValueError if the arguments or configuration passed to action are invalid. action or a hook. Raise ValueError if the arguments or configuration passed to action are
invalid.
''' '''
repository = os.path.expanduser(repository_path) repository = os.path.expanduser(repository_path)
global_arguments = arguments['global'] global_arguments = arguments['global']
dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else ''
hook_context = {
'repository': repository_path,
# Deprecated: For backwards compatibility with borgmatic < 1.6.0.
'repositories': ','.join(location['repositories']),
}
if 'init' in arguments: if 'init' in arguments:
logger.info('{}: Initializing repository'.format(repository)) logger.info('{}: Initializing repository'.format(repository))
borg_init.initialize_repository( borg_init.initialize_repository(
@ -351,6 +262,14 @@ def run_actions(
remote_path=remote_path, remote_path=remote_path,
) )
if 'prune' in arguments: if 'prune' in arguments:
command.execute_hook(
hooks.get('before_prune'),
hooks.get('umask'),
config_filename,
'pre-prune',
global_arguments.dry_run,
**hook_context,
)
logger.info('{}: Pruning archives{}'.format(repository, dry_run_label)) logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
borg_prune.prune_archives( borg_prune.prune_archives(
global_arguments.dry_run, global_arguments.dry_run,
@ -362,7 +281,22 @@ def run_actions(
stats=arguments['prune'].stats, stats=arguments['prune'].stats,
files=arguments['prune'].files, files=arguments['prune'].files,
) )
command.execute_hook(
hooks.get('after_prune'),
hooks.get('umask'),
config_filename,
'post-prune',
global_arguments.dry_run,
**hook_context,
)
if 'compact' in arguments: if 'compact' in arguments:
command.execute_hook(
hooks.get('before_compact'),
hooks.get('umask'),
config_filename,
'pre-compact',
global_arguments.dry_run,
)
if borg_feature.available(borg_feature.Feature.COMPACT, local_borg_version): if borg_feature.available(borg_feature.Feature.COMPACT, local_borg_version):
logger.info('{}: Compacting segments{}'.format(repository, dry_run_label)) logger.info('{}: Compacting segments{}'.format(repository, dry_run_label))
borg_compact.compact_segments( borg_compact.compact_segments(
@ -375,11 +309,26 @@ def run_actions(
cleanup_commits=arguments['compact'].cleanup_commits, cleanup_commits=arguments['compact'].cleanup_commits,
threshold=arguments['compact'].threshold, threshold=arguments['compact'].threshold,
) )
else: else: # pragma: nocover
logger.info( logger.info(
'{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository) '{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository)
) )
command.execute_hook(
hooks.get('after_compact'),
hooks.get('umask'),
config_filename,
'post-compact',
global_arguments.dry_run,
)
if 'create' in arguments: if 'create' in arguments:
command.execute_hook(
hooks.get('before_backup'),
hooks.get('umask'),
config_filename,
'pre-backup',
global_arguments.dry_run,
**hook_context,
)
logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
dispatch.call_hooks( dispatch.call_hooks(
'remove_database_dumps', 'remove_database_dumps',
@ -413,10 +362,35 @@ def run_actions(
files=arguments['create'].files, files=arguments['create'].files,
stream_processes=stream_processes, stream_processes=stream_processes,
) )
if json_output: if json_output: # pragma: nocover
yield json.loads(json_output) yield json.loads(json_output)
dispatch.call_hooks(
'remove_database_dumps',
hooks,
config_filename,
dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
command.execute_hook(
hooks.get('after_backup'),
hooks.get('umask'),
config_filename,
'post-backup',
global_arguments.dry_run,
**hook_context,
)
if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency): if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
command.execute_hook(
hooks.get('before_check'),
hooks.get('umask'),
config_filename,
'pre-check',
global_arguments.dry_run,
**hook_context,
)
logger.info('{}: Running consistency checks'.format(repository)) logger.info('{}: Running consistency checks'.format(repository))
borg_check.check_archives( borg_check.check_archives(
repository, repository,
@ -428,7 +402,23 @@ def run_actions(
repair=arguments['check'].repair, repair=arguments['check'].repair,
only_checks=arguments['check'].only, only_checks=arguments['check'].only,
) )
command.execute_hook(
hooks.get('after_check'),
hooks.get('umask'),
config_filename,
'post-check',
global_arguments.dry_run,
**hook_context,
)
if 'extract' in arguments: if 'extract' in arguments:
command.execute_hook(
hooks.get('before_extract'),
hooks.get('umask'),
config_filename,
'pre-extract',
global_arguments.dry_run,
**hook_context,
)
if arguments['extract'].repository is None or validate.repositories_match( if arguments['extract'].repository is None or validate.repositories_match(
repository, arguments['extract'].repository repository, arguments['extract'].repository
): ):
@ -451,6 +441,14 @@ def run_actions(
strip_components=arguments['extract'].strip_components, strip_components=arguments['extract'].strip_components,
progress=arguments['extract'].progress, progress=arguments['extract'].progress,
) )
command.execute_hook(
hooks.get('after_extract'),
hooks.get('umask'),
config_filename,
'post-extract',
global_arguments.dry_run,
**hook_context,
)
if 'export-tar' in arguments: if 'export-tar' in arguments:
if arguments['export-tar'].repository is None or validate.repositories_match( if arguments['export-tar'].repository is None or validate.repositories_match(
repository, arguments['export-tar'].repository repository, arguments['export-tar'].repository
@ -483,7 +481,7 @@ def run_actions(
logger.info( logger.info(
'{}: Mounting archive {}'.format(repository, arguments['mount'].archive) '{}: Mounting archive {}'.format(repository, arguments['mount'].archive)
) )
else: else: # pragma: nocover
logger.info('{}: Mounting repository'.format(repository)) logger.info('{}: Mounting repository'.format(repository))
borg_mount.mount_archive( borg_mount.mount_archive(
@ -499,7 +497,7 @@ def run_actions(
local_path=local_path, local_path=local_path,
remote_path=remote_path, remote_path=remote_path,
) )
if 'restore' in arguments: if 'restore' in arguments: # pragma: nocover
if arguments['restore'].repository is None or validate.repositories_match( if arguments['restore'].repository is None or validate.repositories_match(
repository, arguments['restore'].repository repository, arguments['restore'].repository
): ):
@ -598,7 +596,7 @@ def run_actions(
repository, arguments['list'].repository repository, arguments['list'].repository
): ):
list_arguments = copy.copy(arguments['list']) list_arguments = copy.copy(arguments['list'])
if not list_arguments.json: if not list_arguments.json: # pragma: nocover
logger.warning('{}: Listing archives'.format(repository)) logger.warning('{}: Listing archives'.format(repository))
list_arguments.archive = borg_list.resolve_archive_name( list_arguments.archive = borg_list.resolve_archive_name(
repository, list_arguments.archive, storage, local_path, remote_path repository, list_arguments.archive, storage, local_path, remote_path
@ -610,14 +608,14 @@ def run_actions(
local_path=local_path, local_path=local_path,
remote_path=remote_path, remote_path=remote_path,
) )
if json_output: if json_output: # pragma: nocover
yield json.loads(json_output) yield json.loads(json_output)
if 'info' in arguments: if 'info' in arguments:
if arguments['info'].repository is None or validate.repositories_match( if arguments['info'].repository is None or validate.repositories_match(
repository, arguments['info'].repository repository, arguments['info'].repository
): ):
info_arguments = copy.copy(arguments['info']) info_arguments = copy.copy(arguments['info'])
if not info_arguments.json: if not info_arguments.json: # pragma: nocover
logger.warning('{}: Displaying summary info for archives'.format(repository)) logger.warning('{}: Displaying summary info for archives'.format(repository))
info_arguments.archive = borg_list.resolve_archive_name( info_arguments.archive = borg_list.resolve_archive_name(
repository, info_arguments.archive, storage, local_path, remote_path repository, info_arguments.archive, storage, local_path, remote_path
@ -629,7 +627,7 @@ def run_actions(
local_path=local_path, local_path=local_path,
remote_path=remote_path, remote_path=remote_path,
) )
if json_output: if json_output: # pragma: nocover
yield json.loads(json_output) yield json.loads(json_output)
if 'borg' in arguments: if 'borg' in arguments:
if arguments['borg'].repository is None or validate.repositories_match( if arguments['borg'].repository is None or validate.repositories_match(

View File

@ -7,11 +7,12 @@ eleventyNavigation:
--- ---
## Preparation and cleanup hooks ## Preparation and cleanup hooks
If you find yourself performing prepraration tasks before your backup runs, or If you find yourself performing preparation tasks before your backup runs, or
cleanup work afterwards, borgmatic hooks may be of interest. Hooks are shell cleanup work afterwards, borgmatic hooks may be of interest. Hooks are shell
commands that borgmatic executes for you at various points, and they're commands that borgmatic executes for you at various points as it runs, and
configured in the `hooks` section of your configuration file. But if you're they're configured in the `hooks` section of your configuration file. But if
looking to backup a database, it's probably easier to use the [database backup you're looking to backup a database, it's probably easier to use the [database
backup
feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
instead. instead.
@ -27,15 +28,14 @@ hooks:
- umount /some/filesystem - umount /some/filesystem
``` ```
The `before_backup` and `after_backup` hooks each run once per configuration The `before_backup` and `after_backup` hooks each run once per repository in a
file. `before_backup` hooks run prior to backups of all repositories in a configuration file. `before_backup` hooks runs right before the `create`
configuration file, right before the `create` action. `after_backup` hooks run action for a particular repository, and `after_backup` hooks run afterwards,
afterwards, but not if an error occurs in a previous hook or in the backups but not if an error occurs in a previous hook or in the backups themselves.
themselves.
There are additional hooks that run before/after other actions as well. For There are additional hooks that run before/after other actions as well. For
instance, `before_prune` runs before a `prune` action, while `after_prune` instance, `before_prune` runs before a `prune` action for a repository, while
runs after it. `after_prune` runs after it.
## Variable interpolation ## Variable interpolation
@ -46,18 +46,18 @@ separate shell script:
```yaml ```yaml
hooks: hooks:
after_prune: after_prune:
- record-prune.sh "{configuration_filename}" "{repositories}" - record-prune.sh "{configuration_filename}" "{repository}"
``` ```
In this example, when the hook is triggered, borgmatic interpolates runtime In this example, when the hook is triggered, borgmatic interpolates runtime
values into the hook command: the borgmatic configuration filename and the values into the hook command: the borgmatic configuration filename and the
paths of all configured repositories. Here's the full set of supported paths of the current Borg repository. Here's the full set of supported
variables you can use here: variables you can use here:
* `configuration_filename`: borgmatic configuration filename in which the * `configuration_filename`: borgmatic configuration filename in which the
hook was defined hook was defined
* `repositories`: comma-separated paths of all repositories configured in the * `repository`: path of the current repository as configured in the current
current borgmatic configuration file borgmatic configuration file
## Global hooks ## Global hooks

View File

@ -1,6 +1,6 @@
from setuptools import find_packages, setup from setuptools import find_packages, setup
VERSION = '1.5.25.dev0' VERSION = '1.6.0.dev0'
setup( setup(

View File

@ -35,75 +35,36 @@ def test_run_configuration_with_invalid_borg_version_errors():
list(module.run_configuration('test.yaml', config, arguments)) list(module.run_configuration('test.yaml', config, arguments))
def test_run_configuration_calls_hooks_for_prune_action(): def test_run_configuration_logs_monitor_start_error():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.command).should_receive('execute_hook').twice() flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return(
flexmock(module.dispatch).should_receive('call_hooks').at_least().twice() None
flexmock(module).should_receive('run_actions').and_return([]) ).and_return(None)
config = {'location': {'repositories': ['foo']}} expected_results = [flexmock()]
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock()} flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').never()
list(module.run_configuration('test.yaml', config, arguments))
def test_run_configuration_calls_hooks_for_compact_action():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.command).should_receive('execute_hook').twice()
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'compact': flexmock()}
list(module.run_configuration('test.yaml', config, arguments))
def test_run_configuration_executes_and_calls_hooks_for_create_action():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.command).should_receive('execute_hook').twice()
flexmock(module.dispatch).should_receive('call_hooks').at_least().twice()
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}} config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_run_configuration_calls_hooks_for_check_action(): def test_run_configuration_bails_for_monitor_start_soft_failure():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.command).should_receive('execute_hook').twice() error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
flexmock(module.dispatch).should_receive('call_hooks').at_least().twice() flexmock(module.dispatch).should_receive('call_hooks').and_raise(error)
flexmock(module).should_receive('run_actions').and_return([]) flexmock(module).should_receive('log_error_records').never()
flexmock(module).should_receive('run_actions').never()
config = {'location': {'repositories': ['foo']}} config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'check': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == []
def test_run_configuration_calls_hooks_for_extract_action():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.command).should_receive('execute_hook').twice()
flexmock(module.dispatch).should_receive('call_hooks').never()
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'extract': flexmock()}
list(module.run_configuration('test.yaml', config, arguments))
def test_run_configuration_does_not_trigger_hooks_for_list_action():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.command).should_receive('execute_hook').never()
flexmock(module.dispatch).should_receive('call_hooks').never()
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'list': flexmock()}
list(module.run_configuration('test.yaml', config, arguments))
def test_run_configuration_logs_actions_error(): def test_run_configuration_logs_actions_error():
@ -122,28 +83,14 @@ def test_run_configuration_logs_actions_error():
assert results == expected_results assert results == expected_results
def test_run_configuration_logs_pre_hook_error(): def test_run_configuration_bails_for_actions_soft_failure():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.command).should_receive('execute_hook').and_raise(OSError).and_return(None)
expected_results = [flexmock()]
flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').never()
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_run_configuration_bails_for_pre_hook_soft_failure():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.dispatch).should_receive('call_hooks')
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
flexmock(module.command).should_receive('execute_hook').and_raise(error).and_return(None) flexmock(module).should_receive('run_actions').and_raise(error)
flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('log_error_records').never()
flexmock(module).should_receive('run_actions').never() flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
config = {'location': {'repositories': ['foo']}} config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
@ -152,13 +99,12 @@ def test_run_configuration_bails_for_pre_hook_soft_failure():
assert results == [] assert results == []
def test_run_configuration_logs_post_hook_error(): def test_run_configuration_logs_monitor_finish_error():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise( flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
OSError None
).and_return(None) ).and_raise(OSError)
flexmock(module.dispatch).should_receive('call_hooks')
expected_results = [flexmock()] expected_results = [flexmock()]
flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_return([]) flexmock(module).should_receive('run_actions').and_return([])
@ -170,16 +116,16 @@ def test_run_configuration_logs_post_hook_error():
assert results == expected_results assert results == expected_results
def test_run_configuration_bails_for_post_hook_soft_failure(): def test_run_configuration_bails_for_monitor_finish_soft_failure():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise( flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
error None
).and_return(None) ).and_raise(error)
flexmock(module.dispatch).should_receive('call_hooks')
flexmock(module).should_receive('log_error_records').never() flexmock(module).should_receive('log_error_records').never()
flexmock(module).should_receive('run_actions').and_return([]) flexmock(module).should_receive('run_actions').and_return([])
flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
config = {'location': {'repositories': ['foo']}} config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
@ -209,7 +155,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(error) flexmock(module.command).should_receive('execute_hook').and_raise(error)
expected_results = [flexmock()] expected_results = [flexmock()]
flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_raise(OSError) flexmock(module).should_receive('run_actions').and_raise(OSError)
@ -411,6 +357,313 @@ def test_run_configuration_retries_timeout_multiple_repos():
assert results == error_logs assert results == error_logs
def test_run_actions_does_not_raise_for_init_action():
flexmock(module.borg_init).should_receive('initialize_repository')
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'init': flexmock(
encryption_mode=flexmock(), append_only=flexmock(), storage_quota=flexmock()
),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_calls_hooks_for_prune_action():
flexmock(module.borg_prune).should_receive('prune_archives')
flexmock(module.command).should_receive('execute_hook').twice()
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'prune': flexmock(stats=flexmock(), files=flexmock()),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_calls_hooks_for_compact_action():
flexmock(module.borg_feature).should_receive('available').and_return(True)
flexmock(module.borg_compact).should_receive('compact_segments')
flexmock(module.command).should_receive('execute_hook').twice()
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'compact': flexmock(progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock()),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_executes_and_calls_hooks_for_create_action():
flexmock(module.borg_create).should_receive('create_archive')
flexmock(module.command).should_receive('execute_hook').twice()
flexmock(module.dispatch).should_receive('call_hooks').and_return({}).times(3)
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'create': flexmock(
progress=flexmock(), stats=flexmock(), json=flexmock(), files=flexmock()
),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_calls_hooks_for_check_action():
flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True)
flexmock(module.borg_check).should_receive('check_archives')
flexmock(module.command).should_receive('execute_hook').twice()
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'check': flexmock(progress=flexmock(), repair=flexmock(), only=flexmock()),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_calls_hooks_for_extract_action():
flexmock(module.validate).should_receive('repositories_match').and_return(True)
flexmock(module.borg_extract).should_receive('extract_archive')
flexmock(module.command).should_receive('execute_hook').twice()
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'extract': flexmock(
paths=flexmock(),
progress=flexmock(),
destination=flexmock(),
strip_components=flexmock(),
archive=flexmock(),
repository='repo',
),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_does_not_raise_for_export_tar_action():
flexmock(module.validate).should_receive('repositories_match').and_return(True)
flexmock(module.borg_export_tar).should_receive('export_tar_archive')
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'export-tar': flexmock(
repository=flexmock(),
archive=flexmock(),
paths=flexmock(),
destination=flexmock(),
tar_filter=flexmock(),
files=flexmock(),
strip_components=flexmock(),
),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_does_not_raise_for_mount_action():
flexmock(module.validate).should_receive('repositories_match').and_return(True)
flexmock(module.borg_mount).should_receive('mount_archive')
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'mount': flexmock(
repository=flexmock(),
archive=flexmock(),
mount_point=flexmock(),
paths=flexmock(),
foreground=flexmock(),
options=flexmock(),
),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_does_not_raise_for_list_action():
flexmock(module.validate).should_receive('repositories_match').and_return(True)
flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock())
flexmock(module.borg_list).should_receive('list_archives')
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'list': flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_does_not_raise_for_info_action():
flexmock(module.validate).should_receive('repositories_match').and_return(True)
flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock())
flexmock(module.borg_info).should_receive('display_archives_info')
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'info': flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_run_actions_does_not_raise_for_borg_action():
flexmock(module.validate).should_receive('repositories_match').and_return(True)
flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock())
flexmock(module.borg_borg).should_receive('run_arbitrary_borg')
arguments = {
'global': flexmock(monitoring_verbosity=1, dry_run=False),
'borg': flexmock(repository=flexmock(), archive=flexmock(), options=flexmock()),
}
list(
module.run_actions(
arguments=arguments,
config_filename='test.yaml',
location={'repositories': ['repo']},
storage={},
retention={},
consistency={},
hooks={},
local_path=None,
remote_path=None,
local_borg_version=None,
repository_path='repo',
)
)
def test_load_configurations_collects_parsed_configurations(): def test_load_configurations_collects_parsed_configurations():
configuration = flexmock() configuration = flexmock()
other_configuration = flexmock() other_configuration = flexmock()