Browse Source

Only check archives with matching prefix.

tags/1.2.0
Nick Whyte 2 years ago
committed by Dan Helfman
parent
commit
c64d0100d5
8 changed files with 84 additions and 13 deletions
  1. +1
    -0
      AUTHORS
  2. +4
    -1
      borgmatic/borg/check.py
  3. +7
    -0
      borgmatic/config/schema.yaml
  4. +7
    -0
      borgmatic/config/validate.py
  5. +31
    -8
      borgmatic/tests/unit/borg/test_check.py
  6. +31
    -1
      borgmatic/tests/unit/config/test_validate.py
  7. +2
    -2
      borgmatic/tests/unit/test_verbosity.py
  8. +1
    -1
      borgmatic/verbosity.py

+ 1
- 0
AUTHORS View File

@@ -8,3 +8,4 @@ newtonne: Read encryption password from external file
Robin `ypid` Schneider: Support additional options of Borg
Scott Squires: Custom archive names
Thomas LÉVEIL: Support for a keep_minutely prune option
Nick Whyte: Support prefix filtering for archive consistency checks

+ 4
- 1
borgmatic/borg/check.py View File

@@ -82,10 +82,13 @@ def check_archives(verbosity, repository, storage_config, consistency_config, lo
VERBOSITY_LOTS: ('--debug',),
}.get(verbosity, ())

prefix = consistency_config.get('prefix', '{hostname}-')
prefix_flags = ('--prefix', prefix) if prefix else ()

full_command = (
local_path, 'check',
repository,
) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags
) + _make_check_flags(checks, check_last) + prefix_flags + remote_path_flags + lock_wait_flags + verbosity_flags

# The check command spews to stdout/stderr even without the verbose flag. Suppress it.
stdout = None if verbosity_flags else open(os.devnull, 'w')


+ 7
- 0
borgmatic/config/schema.yaml View File

@@ -218,6 +218,13 @@ map:
desc: Restrict the number of checked archives to the last n. Applies only to the
"archives" check.
example: 3
prefix:
type: scalar
desc: |
When performing consistency checks, only consider archive names starting with
this prefix. Borg placeholders can be used. See the output of
"borg help placeholders" for details. Default is "{hostname}-".
example: sourcehostname
hooks:
desc: |
Shell commands or scripts to execute before and after a backup or if an error has occurred.


+ 7
- 0
borgmatic/config/validate.py View File

@@ -8,6 +8,8 @@ import pykwalify.errors
from ruamel import yaml


logger = logging.getLogger(__name__)

def schema_filename():
'''
Path to the installed YAML configuration schema file, used to validate and parse the
@@ -50,6 +52,11 @@ def apply_logical_validation(config_filename, parsed_configuration):
)
)

consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
if archive_name_format and not consistency_prefix:
logger.warning('Since version 1.2.0, if you provide `archive_name_format`, you must also'
' specify `consistency.prefix`.')


def parse_configuration(config_filename, schema_filename):
'''


+ 31
- 8
borgmatic/tests/unit/borg/test_check.py View File

@@ -89,7 +89,7 @@ def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
)
def test_check_archives_calls_borg_with_parameters(checks):
check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock()
@@ -111,7 +111,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
def test_check_archives_with_extract_check_calls_extract_only():
checks = ('extract',)
check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').never()
flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
@@ -127,7 +127,7 @@ def test_check_archives_with_extract_check_calls_extract_only():

def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
checks = ('repository',)
consistency_config = flexmock().should_receive('get').and_return(None).mock
consistency_config = {'check_last': None}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock(
@@ -145,7 +145,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():

def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
checks = ('repository',)
consistency_config = flexmock().should_receive('get').and_return(None).mock
consistency_config = {'check_last': None}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock(
@@ -162,7 +162,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():


def test_check_archives_without_any_checks_bails():
consistency_config = flexmock().should_receive('get').and_return(None).mock
consistency_config = {'check_last': None}
flexmock(module).should_receive('_parse_checks').and_return(())
insert_subprocess_never()

@@ -177,7 +177,7 @@ def test_check_archives_without_any_checks_bails():
def test_check_archives_with_local_path_calls_borg_via_local_path():
checks = ('repository',)
check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock()
@@ -200,7 +200,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters():
checks = ('repository',)
check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock()
@@ -223,7 +223,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
checks = ('repository',)
check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock()
@@ -240,3 +240,26 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config={'lock_wait': 5},
consistency_config=consistency_config,
)


def test_check_archives_with_retention_prefix():
checks = ('repository',)
check_last = flexmock()
consistency_config = {'check_last': check_last, 'prefix': 'foo-'}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock()
insert_subprocess_mock(
('borg', 'check', 'repo', '--prefix', 'foo-'),
stdout=stdout, stderr=STDOUT,
)

flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')

module.check_archives(
verbosity=None,
repository='repo',
storage_config={},
consistency_config=consistency_config,
)

+ 31
- 1
borgmatic/tests/unit/config/test_validate.py View File

@@ -1,4 +1,5 @@
import pytest
from flexmock import flexmock

from borgmatic.config import validate as module

@@ -23,13 +24,42 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
},
)

def test_apply_logical_validation_raises_if_archive_name_format_present_without_retention_prefix():
with pytest.raises(module.Validation_error):
module.apply_logical_validation(
'config.yaml',
{
'storage': {'archive_name_format': '{hostname}-{now}'},
'retention': {'keep_daily': 7},
'consistency': {'prefix': '{hostname}-'}
},
)


def test_apply_logical_validation_warns_if_archive_name_format_present_without_consistency_prefix():
logger = flexmock(module.logger)
logger.should_receive('warning').once()

module.apply_logical_validation(
'config.yaml',
{
'storage': {'archive_name_format': '{hostname}-{now}'},
'retention': {'prefix': '{hostname}-'},
'consistency': {},
},
)


def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present():
logger = flexmock(module.logger)
logger.should_receive('warning').never()

def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present():
module.apply_logical_validation(
'config.yaml',
{
'storage': {'archive_name_format': '{hostname}-{now}'},
'retention': {'prefix': '{hostname}-'},
'consistency': {'prefix': '{hostname}-'}
},
)



+ 2
- 2
borgmatic/tests/unit/test_verbosity.py View File

@@ -7,5 +7,5 @@ def test_verbosity_to_log_level_maps_known_verbosity_to_log_level():
assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO


def test_verbosity_to_log_level_maps_unknown_verbosity_to_error_level():
assert module.verbosity_to_log_level('my pants') == logging.ERROR
def test_verbosity_to_log_level_maps_unknown_verbosity_to_warning_level():
assert module.verbosity_to_log_level('my pants') == logging.WARNING

+ 1
- 1
borgmatic/verbosity.py View File

@@ -12,4 +12,4 @@ def verbosity_to_log_level(verbosity):
return {
VERBOSITY_SOME: logging.INFO,
VERBOSITY_LOTS: logging.DEBUG,
}.get(verbosity, logging.ERROR)
}.get(verbosity, logging.WARNING)

Loading…
Cancel
Save