diff --git a/NEWS b/NEWS index e093afc4..012584d1 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,9 @@ 1.8.5.dev0 + * #701: Add a "skip_actions" option to skip running particular actions, handy for append-only or + checkless configurations. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/set-up-backups/#skipping-actions + * #701: Deprecate the "disabled" value for the "checks" option in favor of the new "skip_actions" + option. * #779: Add a "--match-archives" flag to the "check" action for selecting the archives to check, overriding the existing "archive_name_format" and "match_archives" options in configuration. * #779: Only parse "--override" values as complex data types when they're for options of those diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 14d7a827..220a0fc2 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -39,7 +39,11 @@ def parse_checks(config, only_checks=None): check_config['name'] for check_config in (config.get('checks', None) or DEFAULT_CHECKS) ) checks = tuple(check.lower() for check in checks) + if 'disabled' in checks: + logger.warning( + 'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead' + ) if len(checks) > 1: logger.warning( 'Multiple checks are configured, but one of them is "disabled"; not running any checks' @@ -119,6 +123,9 @@ def filter_checks_on_frequency( Raise ValueError if a frequency cannot be parsed. ''' + if not checks: + return checks + filtered_checks = list(checks) if force: diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index ba80d2c0..ee03ddef 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -70,6 +70,12 @@ def run_configuration(config_filename, config, arguments): using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED + skip_actions = config.get('skip_actions') + + if skip_actions: + logger.debug( + f"{config_filename}: Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions" + ) try: local_borg_version = borg_version.local_borg_version(config, local_path) @@ -274,6 +280,7 @@ def run_actions( 'repositories': ','.join([repo['path'] for repo in config['repositories']]), 'log_file': global_arguments.log_file if global_arguments.log_file else '', } + skip_actions = set(config.get('skip_actions', {})) command.execute_hook( config.get('before_actions'), @@ -285,7 +292,7 @@ def run_actions( ) for action_name, action_arguments in arguments.items(): - if action_name == 'rcreate': + if action_name == 'rcreate' and action_name not in skip_actions: borgmatic.actions.rcreate.run_rcreate( repository, config, @@ -295,7 +302,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'transfer': + elif action_name == 'transfer' and action_name not in skip_actions: borgmatic.actions.transfer.run_transfer( repository, config, @@ -305,7 +312,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'create': + elif action_name == 'create' and action_name not in skip_actions: yield from borgmatic.actions.create.run_create( config_filename, repository, @@ -318,7 +325,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'prune': + elif action_name == 'prune' and action_name not in skip_actions: borgmatic.actions.prune.run_prune( config_filename, repository, @@ -331,7 +338,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'compact': + elif action_name == 'compact' and action_name not in skip_actions: borgmatic.actions.compact.run_compact( config_filename, repository, @@ -344,7 +351,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'check': + elif action_name == 'check' and action_name not in skip_actions: if checks.repository_enabled_for_checks(repository, config): borgmatic.actions.check.run_check( config_filename, @@ -357,7 +364,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'extract': + elif action_name == 'extract' and action_name not in skip_actions: borgmatic.actions.extract.run_extract( config_filename, repository, @@ -369,7 +376,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'export-tar': + elif action_name == 'export-tar' and action_name not in skip_actions: borgmatic.actions.export_tar.run_export_tar( repository, config, @@ -379,7 +386,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'mount': + elif action_name == 'mount' and action_name not in skip_actions: borgmatic.actions.mount.run_mount( repository, config, @@ -389,7 +396,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'restore': + elif action_name == 'restore' and action_name not in skip_actions: borgmatic.actions.restore.run_restore( repository, config, @@ -399,7 +406,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'rlist': + elif action_name == 'rlist' and action_name not in skip_actions: yield from borgmatic.actions.rlist.run_rlist( repository, config, @@ -409,7 +416,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'list': + elif action_name == 'list' and action_name not in skip_actions: yield from borgmatic.actions.list.run_list( repository, config, @@ -419,7 +426,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'rinfo': + elif action_name == 'rinfo' and action_name not in skip_actions: yield from borgmatic.actions.rinfo.run_rinfo( repository, config, @@ -429,7 +436,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'info': + elif action_name == 'info' and action_name not in skip_actions: yield from borgmatic.actions.info.run_info( repository, config, @@ -439,7 +446,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'break-lock': + elif action_name == 'break-lock' and action_name not in skip_actions: borgmatic.actions.break_lock.run_break_lock( repository, config, @@ -449,7 +456,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'export': + elif action_name == 'export' and action_name not in skip_actions: borgmatic.actions.export_key.run_export_key( repository, config, @@ -459,7 +466,7 @@ def run_actions( local_path, remote_path, ) - elif action_name == 'borg': + elif action_name == 'borg' and action_name not in skip_actions: borgmatic.actions.borg.run_borg( repository, config, diff --git a/borgmatic/config/checks.py b/borgmatic/config/checks.py index 13361ea1..129fd61b 100644 --- a/borgmatic/config/checks.py +++ b/borgmatic/config/checks.py @@ -1,9 +1,9 @@ -def repository_enabled_for_checks(repository, consistency): +def repository_enabled_for_checks(repository, config): ''' - Given a repository name and a consistency configuration dict, return whether the repository - is enabled to have consistency checks run. + Given a repository name and a configuration dict, return whether the + repository is enabled to have consistency checks run. ''' - if not consistency.get('check_repositories'): + if not config.get('check_repositories'): return True - return repository in consistency['check_repositories'] + return repository in config['check_repositories'] diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 41ce4c73..b1522f3a 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -423,7 +423,9 @@ properties: command-line invocation. keep_within: type: string - description: Keep all archives within this time interval. + description: | + Keep all archives within this time interval. See "skip_actions" for + disabling pruning altogether. example: 3H keep_secondly: type: integer @@ -479,13 +481,13 @@ properties: - disabled description: | Name of consistency check to run: "repository", - "archives", "data", and/or "extract". Set to "disabled" - to disable all consistency checks. "repository" checks - the consistency of the repository, "archives" checks all - of the archives, "data" verifies the integrity of the - data within the archives, and "extract" does an - extraction dry-run of the most recent archive. Note that - "data" implies "archives". + "archives", "data", and/or "extract". "repository" + checks the consistency of the repository, "archives" + checks all of the archives, "data" verifies the + integrity of the data within the archives, and "extract" + does an extraction dry-run of the most recent archive. + Note that "data" implies "archives". See "skip_actions" + for disabling checks altogether. example: repository frequency: type: string @@ -525,6 +527,18 @@ properties: Apply color to console output. Can be overridden with --no-color command-line flag. Defaults to true. example: false + skip_actions: + type: array + items: + type: string + description: | + List of one or more actions to skip running for this configuration + file, even if specified on the command-line (explicitly or + implicitly). This is handy for append-only configurations where you + never want to run "compact" or checkless configuration where you + want to skip "check". Defaults to not skipping any actions. + example: + - compact before_actions: type: array items: diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index c724581f..c609ba19 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -162,7 +162,16 @@ location: If that's still too slow, you can disable consistency checks entirely, either for a single repository or for all repositories. -Disabling all consistency checks looks like this: +New in version 1.8.5 Disabling +all consistency checks looks like this: + +```yaml +skip_actions: + - check +``` + +Prior to version 1.8.5 Use this +configuration instead: ```yaml checks: @@ -170,10 +179,10 @@ checks: ``` Prior to version 1.8.0 Put -this option in the `consistency:` section of your configuration. +`checks:` in the `consistency:` section of your configuration. -Prior to version 1.6.2 `checks` -was a plain list of strings without the `name:` part. For instance: +Prior to version 1.6.2 +`checks:` was a plain list of strings without the `name:` part. For instance: ```yaml checks: diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 1e05bde7..22554dd4 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -282,6 +282,21 @@ due to things like file damage. For instance: sudo borgmatic --verbosity 1 --list --stats ``` +### Skipping actions + +New in version 1.8.5 You can +configure borgmatic to skip running certain actions (default or otherwise). +For instance, to always skip the `compact` action when using [Borg's +append-only +mode](https://borgbackup.readthedocs.io/en/stable/usage/notes.html#append-only-mode-forbid-compaction), +set the `skip_actions` option: + +``` +skip_actions: + - compact +``` + + ## Autopilot Running backups manually is good for validating your configuration, but I'm diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 8d8ac45d..e222d680 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -193,6 +193,19 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_ ) == ('archives',) +def test_filter_checks_on_frequency_passes_through_empty_checks(): + assert ( + module.filter_checks_on_frequency( + config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]}, + borg_repository_id='repo', + checks=(), + force=False, + archives_check_id='1234', + ) + == () + ) + + def test_make_archive_filter_flags_with_default_checks_and_prefix_returns_default_flags(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 26321c31..d7334f5c 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -23,6 +23,16 @@ def test_run_configuration_runs_actions_for_each_repository(): assert results == expected_results +def test_run_configuration_with_skip_actions_does_not_raise(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) + flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) + flexmock(module).should_receive('run_actions').and_return(flexmock()).and_return(flexmock()) + config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'skip_actions': ['compact']} + arguments = {'global': flexmock(monitoring_verbosity=1)} + + list(module.run_configuration('test.yaml', config, arguments)) + + def test_run_configuration_with_invalid_borg_version_errors(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError) @@ -504,6 +514,24 @@ def test_run_actions_runs_create(): assert result == (expected,) +def test_run_actions_with_skip_actions_skips_create(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.create).should_receive('run_create').never() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, + config_filename=flexmock(), + config={'repositories': [], 'skip_actions': ['create']}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + def test_run_actions_runs_prune(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') @@ -522,6 +550,24 @@ def test_run_actions_runs_prune(): ) +def test_run_actions_with_skip_actions_skips_prune(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.prune).should_receive('run_prune').never() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'prune': flexmock()}, + config_filename=flexmock(), + config={'repositories': [], 'skip_actions': ['prune']}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + def test_run_actions_runs_compact(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') @@ -540,6 +586,24 @@ def test_run_actions_runs_compact(): ) +def test_run_actions_with_skip_actions_skips_compact(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + flexmock(borgmatic.actions.compact).should_receive('run_compact').never() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'compact': flexmock()}, + config_filename=flexmock(), + config={'repositories': [], 'skip_actions': ['compact']}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + def test_run_actions_runs_check_when_repository_enabled_for_checks(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') @@ -578,6 +642,25 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): ) +def test_run_actions_with_skip_actions_skips_check(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module.command).should_receive('execute_hook') + flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True) + flexmock(borgmatic.actions.check).should_receive('run_check').never() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()}, + config_filename=flexmock(), + config={'repositories': [], 'skip_actions': ['check']}, + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + def test_run_actions_runs_extract(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.command).should_receive('execute_hook') diff --git a/tests/unit/config/test_checks.py b/tests/unit/config/test_checks.py index df6df1ec..6de42b4b 100644 --- a/tests/unit/config/test_checks.py +++ b/tests/unit/config/test_checks.py @@ -2,14 +2,14 @@ from borgmatic.config import checks as module def test_repository_enabled_for_checks_defaults_to_enabled_for_all_repositories(): - enabled = module.repository_enabled_for_checks('repo.borg', consistency={}) + enabled = module.repository_enabled_for_checks('repo.borg', config={}) assert enabled def test_repository_enabled_for_checks_is_enabled_for_specified_repositories(): enabled = module.repository_enabled_for_checks( - 'repo.borg', consistency={'check_repositories': ['repo.borg', 'other.borg']} + 'repo.borg', config={'check_repositories': ['repo.borg', 'other.borg']} ) assert enabled @@ -17,7 +17,7 @@ def test_repository_enabled_for_checks_is_enabled_for_specified_repositories(): def test_repository_enabled_for_checks_is_disabled_for_other_repositories(): enabled = module.repository_enabled_for_checks( - 'repo.borg', consistency={'check_repositories': ['other.borg']} + 'repo.borg', config={'check_repositories': ['other.borg']} ) assert not enabled