Fix archive checks being skipped even when particular archives haven't been checked recently (#688).
Some checks failed
continuous-integration/drone/push Build is failing

This commit is contained in:
Dan Helfman 2023-05-15 23:17:45 -07:00
parent e66e449c3b
commit 645d29b040
6 changed files with 587 additions and 270 deletions

3
NEWS
View File

@ -16,6 +16,9 @@
https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion
* #687: Fix borgmatic error when not finding the configuration schema for certain "pip install
--editable" development installs.
* #688: Fix archive checks being skipped even when particular archives haven't been checked
recently. This occurred when using multiple borgmatic configuration files with different
"archive_name_format"s, for instance.
* #691: Fix error in "borgmatic restore" action when the configured repository path is relative
instead of absolute.
* #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like

View File

@ -1,5 +1,7 @@
import argparse
import datetime
import hashlib
import itertools
import json
import logging
import os
@ -88,12 +90,18 @@ def parse_frequency(frequency):
def filter_checks_on_frequency(
location_config, consistency_config, borg_repository_id, checks, force
location_config,
consistency_config,
borg_repository_id,
checks,
force,
archives_check_id=None,
):
'''
Given a location config, a consistency config with a "checks" sequence of dicts, a Borg
repository ID, a sequence of checks, and whether to force checks to run, filter down those
checks based on the configured "frequency" for each check as compared to its check time file.
repository ID, a sequence of checks, whether to force checks to run, and an ID for the archives
check potentially being run (if any), filter down those checks based on the configured
"frequency" for each check as compared to its check time file.
In other words, a check whose check time file's timestamp is too new (based on the configured
frequency) will get cut from the returned sequence of checks. Example:
@ -127,8 +135,8 @@ def filter_checks_on_frequency(
if not frequency_delta:
continue
check_time = read_check_time(
make_check_time_path(location_config, borg_repository_id, check)
check_time = probe_for_check_time(
location_config, borg_repository_id, check, archives_check_id
)
if not check_time:
continue
@ -145,36 +153,19 @@ def filter_checks_on_frequency(
return tuple(filtered_checks)
def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None):
def make_archive_filter_flags(
local_borg_version, storage_config, checks, check_last=None, prefix=None
):
'''
Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the
check last value, and a consistency check prefix, transform the checks into tuple of
command-line flags.
command-line flags for filtering archives in a check command.
For example, given parsed checks of:
('repository',)
This will be returned as:
('--repository-only',)
However, if both "repository" and "archives" are in checks, then omit them from the returned
flags because Borg does both checks by default. If "data" is in checks, that implies "archives".
Additionally, if a check_last value is given and "archives" is in checks, then include a
"--last" flag. And if a prefix value is given and "archives" is in checks, then include a
"--match-archives" flag.
If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if
a prefix value is given and "archives" is in checks, then include a "--match-archives" flag.
'''
if 'data' in checks:
data_flags = ('--verify-data',)
checks += ('archives',)
else:
data_flags = ()
if 'archives' in checks:
last_flags = ('--last', str(check_last)) if check_last else ()
match_archives_flags = (
if 'archives' in checks or 'data' in checks:
return (('--last', str(check_last)) if check_last else ()) + (
(
('--match-archives', f'sh:{prefix}*')
if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
@ -189,19 +180,53 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None
)
)
)
else:
last_flags = ()
match_archives_flags = ()
if check_last:
logger.warning(
'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
)
if prefix:
logger.warning(
'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
)
common_flags = last_flags + match_archives_flags + data_flags
if check_last:
logger.warning(
'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
)
if prefix:
logger.warning(
'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
)
return ()
def make_archives_check_id(archive_filter_flags):
'''
Given a sequence of flags to filter archives, return a unique hash corresponding to those
particular flags. If there are no flags, return None.
'''
if not archive_filter_flags:
return None
return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest()
def make_check_flags(checks, archive_filter_flags):
'''
Given a parsed sequence of checks and a sequence of flags to filter archives, transform the
checks into tuple of command-line check flags.
For example, given parsed checks of:
('repository',)
This will be returned as:
('--repository-only',)
However, if both "repository" and "archives" are in checks, then omit them from the returned
flags because Borg does both checks by default. If "data" is in checks, that implies "archives".
'''
if 'data' in checks:
data_flags = ('--verify-data',)
checks += ('archives',)
else:
data_flags = ()
common_flags = archive_filter_flags + data_flags
if {'repository', 'archives'}.issubset(set(checks)):
return common_flags
@ -212,18 +237,27 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None
)
def make_check_time_path(location_config, borg_repository_id, check_type):
def make_check_time_path(location_config, borg_repository_id, check_type, archives_check_id=None):
'''
Given a location configuration dict, a Borg repository ID, and the name of a check type
("repository", "archives", etc.), return a path for recording that check's time (the time of
that check last occurring).
Given a location configuration dict, a Borg repository ID, the name of a check type
("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
path for recording that check's time (the time of that check last occurring).
'''
borgmatic_source_directory = os.path.expanduser(
location_config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY)
)
if check_type in ('archives', 'data'):
return os.path.join(
borgmatic_source_directory,
'checks',
borg_repository_id,
check_type,
archives_check_id if archives_check_id else 'all',
)
return os.path.join(
os.path.expanduser(
location_config.get(
'borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
)
),
borgmatic_source_directory,
'checks',
borg_repository_id,
check_type,
@ -253,6 +287,74 @@ def read_check_time(path):
return None
def probe_for_check_time(location_config, borg_repository_id, check, archives_check_id):
'''
Given a location configuration dict, a Borg repository ID, the name of a check type
("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
the corresponding check time or None if such a check time does not exist.
When the check type is "archives" or "data", this function probes two different paths to find
the check time, e.g.:
~/.borgmatic/checks/1234567890/archives/9876543210
~/.borgmatic/checks/1234567890/archives/all
... and returns the modification time of the first file found (if any). The first path
represents a more specific archives check time (a check on a subset of archives), and the second
is a fallback to the last "all" archives check.
For other check types, this function reads from a single check time path, e.g.:
~/.borgmatic/checks/1234567890/repository
'''
check_times = (
read_check_time(group[0])
for group in itertools.groupby(
(
make_check_time_path(location_config, borg_repository_id, check, archives_check_id),
make_check_time_path(location_config, borg_repository_id, check),
)
)
)
try:
return next(check_time for check_time in check_times if check_time)
except StopIteration:
return None
def upgrade_check_times(location_config, borg_repository_id):
'''
Given a location configuration dict and a Borg repository ID, upgrade any corresponding check
times on disk from old-style paths to new-style paths.
Currently, the only upgrade performed is renaming an archive or data check path that looks like:
~/.borgmatic/checks/1234567890/archives
to:
~/.borgmatic/checks/1234567890/archives/all
'''
for check_type in ('archives', 'data'):
new_path = make_check_time_path(location_config, borg_repository_id, check_type, 'all')
old_path = os.path.dirname(new_path)
temporary_path = f'{old_path}.temp'
if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
return
logger.debug(f'Upgrading archives check time from {old_path} to {new_path}')
try:
os.rename(old_path, temporary_path)
except FileNotFoundError:
pass
os.mkdir(old_path)
os.rename(temporary_path, new_path)
def check_archives(
repository_path,
location_config,
@ -292,16 +394,26 @@ def check_archives(
except (json.JSONDecodeError, KeyError):
raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
upgrade_check_times(location_config, borg_repository_id)
check_last = consistency_config.get('check_last', None)
prefix = consistency_config.get('prefix')
configured_checks = parse_checks(consistency_config, only_checks)
lock_wait = None
extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
archive_filter_flags = make_archive_filter_flags(
local_borg_version, storage_config, configured_checks, check_last, prefix
)
archives_check_id = make_archives_check_id(archive_filter_flags)
checks = filter_checks_on_frequency(
location_config,
consistency_config,
borg_repository_id,
parse_checks(consistency_config, only_checks),
configured_checks,
force,
archives_check_id,
)
check_last = consistency_config.get('check_last', None)
lock_wait = None
extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
if set(checks).intersection({'repository', 'archives', 'data'}):
lock_wait = storage_config.get('lock_wait')
@ -312,12 +424,10 @@ def check_archives(
if logger.isEnabledFor(logging.DEBUG):
verbosity_flags = ('--debug', '--show-rc')
prefix = consistency_config.get('prefix')
full_command = (
(local_path, 'check')
+ (('--repair',) if repair else ())
+ make_check_flags(local_borg_version, storage_config, checks, check_last, prefix)
+ make_check_flags(checks, archive_filter_flags)
+ (('--remote-path', remote_path) if remote_path else ())
+ (('--log-json',) if global_arguments.log_json else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
@ -339,7 +449,9 @@ def check_archives(
execute_command(full_command, extra_environment=borg_environment)
for check in checks:
write_check_time(make_check_time_path(location_config, borg_repository_id, check))
write_check_time(
make_check_time_path(location_config, borg_repository_id, check, archives_check_id)
)
if 'extract' in checks:
extract.extract_last_archive_dry_run(

View File

@ -95,6 +95,7 @@ See [Borg's check
documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html)
for more information.
### Check frequency
<span class="minilink minilink-addedin">New in version 1.6.2</span> You can

View File

@ -81,6 +81,9 @@ If `archive_name_format` is unspecified, the default is
`{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a
timestamp in a particular format.
### Achive filtering
<span class="minilink minilink-addedin">New in version 1.7.11</span> borgmatic
uses the `archive_name_format` option to automatically limit which archives
get used for actions operating on multiple archives. This prevents, for

View File

@ -7,7 +7,7 @@ from borgmatic.borg import borg as module
from ..test_verbosity import insert_logging_mock
def test_run_arbitrary_borg_calls_borg_with_parameters():
def test_run_arbitrary_borg_calls_borg_with_flags():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -28,7 +28,7 @@ def test_run_arbitrary_borg_calls_borg_with_parameters():
)
def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter():
def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -50,7 +50,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter():
)
def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter():
def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -72,7 +72,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter():
)
def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters():
def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
storage_config = {'lock_wait': 5}
@ -96,7 +96,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters(
)
def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter():
def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
@ -142,7 +142,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
)
def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_parameters():
def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -166,7 +166,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet
)
def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg():
def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -187,7 +187,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg():
)
def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg():
def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -208,7 +208,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg():
)
def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise():
def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_repository_flags').never()

View File

@ -96,7 +96,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks():
module.datetime.timedelta(weeks=4)
)
flexmock(module).should_receive('make_check_time_path')
flexmock(module).should_receive('read_check_time').and_return(None)
flexmock(module).should_receive('probe_for_check_time').and_return(None)
assert module.filter_checks_on_frequency(
location_config={},
@ -104,6 +104,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks():
borg_repository_id='repo',
checks=('repository', 'archives'),
force=False,
archives_check_id='1234',
) == ('repository', 'archives')
@ -126,6 +127,7 @@ def test_filter_checks_on_frequency_retains_check_without_frequency():
borg_repository_id='repo',
checks=('archives',),
force=False,
archives_check_id='1234',
) == ('archives',)
@ -134,7 +136,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency():
module.datetime.timedelta(hours=1)
)
flexmock(module).should_receive('make_check_time_path')
flexmock(module).should_receive('read_check_time').and_return(
flexmock(module).should_receive('probe_for_check_time').and_return(
module.datetime.datetime(year=module.datetime.MINYEAR, month=1, day=1)
)
@ -144,6 +146,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency():
borg_repository_id='repo',
checks=('archives',),
force=False,
archives_check_id='1234',
) == ('archives',)
@ -152,7 +155,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file()
module.datetime.timedelta(hours=1)
)
flexmock(module).should_receive('make_check_time_path')
flexmock(module).should_receive('read_check_time').and_return(None)
flexmock(module).should_receive('probe_for_check_time').and_return(None)
assert module.filter_checks_on_frequency(
location_config={},
@ -160,6 +163,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file()
borg_repository_id='repo',
checks=('archives',),
force=False,
archives_check_id='1234',
) == ('archives',)
@ -168,7 +172,9 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency():
module.datetime.timedelta(hours=1)
)
flexmock(module).should_receive('make_check_time_path')
flexmock(module).should_receive('read_check_time').and_return(module.datetime.datetime.now())
flexmock(module).should_receive('probe_for_check_time').and_return(
module.datetime.datetime.now()
)
assert (
module.filter_checks_on_frequency(
@ -177,6 +183,7 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency():
borg_repository_id='repo',
checks=('archives',),
force=False,
archives_check_id='1234',
)
== ()
)
@ -189,32 +196,177 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_
borg_repository_id='repo',
checks=('archives',),
force=True,
archives_check_id='1234',
) == ('archives',)
def test_make_check_flags_with_repository_check_returns_flag():
def test_make_archive_filter_flags_with_default_checks_and_prefix_returns_default_flags():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('repository',))
flags = module.make_archive_filter_flags(
'1.2.3',
{},
('repository', 'archives'),
prefix='foo',
)
assert flags == ('--match-archives', 'sh:foo*')
def test_make_archive_filter_flags_with_all_checks_and_prefix_returns_default_flags():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags(
'1.2.3',
{},
('repository', 'archives', 'extract'),
prefix='foo',
)
assert flags == ('--match-archives', 'sh:foo*')
def test_make_archive_filter_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags():
flexmock(module.feature).should_receive('available').and_return(False)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags(
'1.2.3',
{},
('repository', 'archives', 'extract'),
prefix='foo',
)
assert flags == ('--glob-archives', 'foo*')
def test_make_archive_filter_flags_with_archives_check_and_last_includes_last_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), check_last=3)
assert flags == ('--last', '3')
def test_make_archive_filter_flags_with_data_check_and_last_includes_last_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), check_last=3)
assert flags == ('--last', '3')
def test_make_archive_filter_flags_with_repository_check_and_last_omits_last_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), check_last=3)
assert flags == ()
def test_make_archive_filter_flags_with_default_checks_and_last_includes_last_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), check_last=3)
assert flags == ('--last', '3')
def test_make_archive_filter_flags_with_archives_check_and_prefix_includes_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix='foo-')
assert flags == ('--match-archives', 'sh:foo-*')
def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), prefix='foo-')
assert flags == ('--match-archives', 'sh:foo-*')
def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, 'bar-{now}', '1.2.3' # noqa: FS003
).and_return(('--match-archives', 'sh:bar-*'))
flags = module.make_archive_filter_flags(
'1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003
)
assert flags == ('--match-archives', 'sh:bar-*')
def test_make_archive_filter_flags_with_archives_check_and_none_prefix_omits_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix=None)
assert flags == ()
def test_make_archive_filter_flags_with_repository_check_and_prefix_omits_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), prefix='foo-')
assert flags == ()
def test_make_archive_filter_flags_with_default_checks_and_prefix_includes_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-')
assert flags == ('--match-archives', 'sh:foo-*')
def test_make_archives_check_id_with_flags_returns_a_value_and_does_not_raise():
assert module.make_archives_check_id(('--match-archives', 'sh:foo-*'))
def test_make_archives_check_id_with_empty_flags_returns_none():
assert module.make_archives_check_id(()) is None
def test_make_check_flags_with_repository_check_returns_flag():
flags = module.make_check_flags(('repository',), ())
assert flags == ('--repository-only',)
def test_make_check_flags_with_archives_check_returns_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('archives',))
flags = module.make_check_flags(('archives',), ())
assert flags == ('--archives-only',)
def test_make_check_flags_with_archive_filtler_flags_includes_those_flags():
flags = module.make_check_flags(('archives',), ('--match-archives', 'sh:foo-*'))
assert flags == ('--archives-only', '--match-archives', 'sh:foo-*')
def test_make_check_flags_with_data_check_returns_flag_and_implies_archives():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('data',))
flags = module.make_check_flags(('data',), ())
assert flags == (
'--archives-only',
@ -226,7 +378,7 @@ def test_make_check_flags_with_extract_omits_extract_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('extract',))
flags = module.make_check_flags(('extract',), ())
assert flags == ()
@ -236,151 +388,66 @@ def test_make_check_flags_with_repository_and_data_checks_does_not_return_reposi
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags(
'1.2.3',
{},
(
'repository',
'data',
),
(),
)
assert flags == ('--verify-data',)
def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags(
'1.2.3',
{},
('repository', 'archives'),
prefix='foo',
def test_make_check_time_path_with_borgmatic_source_directory_includes_it():
flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return(
'/home/user/.borgmatic'
)
assert flags == ('--match-archives', 'sh:foo*')
def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags(
'1.2.3',
{},
('repository', 'archives', 'extract'),
prefix='foo',
assert (
module.make_check_time_path(
{'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'archives', '5678'
)
== '/home/user/.borgmatic/checks/1234/archives/5678'
)
assert flags == ('--match-archives', 'sh:foo*')
def test_make_check_time_path_without_borgmatic_source_directory_uses_default():
flexmock(module.os.path).should_receive('expanduser').with_args(
module.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
).and_return('/home/user/.borgmatic')
def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags():
flexmock(module.feature).should_receive('available').and_return(False)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags(
'1.2.3',
{},
('repository', 'archives', 'extract'),
prefix='foo',
assert (
module.make_check_time_path({}, '1234', 'archives', '5678')
== '/home/user/.borgmatic/checks/1234/archives/5678'
)
assert flags == ('--glob-archives', 'foo*')
def test_make_check_flags_with_archives_check_and_last_includes_last_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('archives',), check_last=3)
assert flags == ('--archives-only', '--last', '3')
def test_make_check_flags_with_data_check_and_last_includes_last_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('data',), check_last=3)
assert flags == ('--archives-only', '--last', '3', '--verify-data')
def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('repository',), check_last=3)
assert flags == ('--repository-only',)
def test_make_check_flags_with_default_checks_and_last_includes_last_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), check_last=3)
assert flags == ('--last', '3')
def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix='foo-')
assert flags == ('--archives-only', '--match-archives', 'sh:foo-*')
def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('data',), prefix='foo-')
assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data')
def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
None, 'bar-{now}', '1.2.3' # noqa: FS003
).and_return(('--match-archives', 'sh:bar-*'))
flags = module.make_check_flags(
'1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003
def test_make_check_time_path_with_archives_check_and_no_archives_check_id_defaults_to_all():
flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return(
'/home/user/.borgmatic'
)
assert flags == ('--archives-only', '--match-archives', 'sh:bar-*')
assert (
module.make_check_time_path(
{'borgmatic_source_directory': '~/.borgmatic'},
'1234',
'archives',
)
== '/home/user/.borgmatic/checks/1234/archives/all'
)
def test_make_check_flags_with_archives_check_and_none_prefix_omits_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
def test_make_check_time_path_with_repositories_check_ignores_archives_check_id():
flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return(
'/home/user/.borgmatic'
)
flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix=None)
assert flags == ('--archives-only',)
def test_make_check_flags_with_repository_check_and_prefix_omits_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('repository',), prefix='foo-')
assert flags == ('--repository-only',)
def test_make_check_flags_with_default_checks_and_prefix_includes_match_archives_flag():
flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-')
assert flags == ('--match-archives', 'sh:foo-*')
assert (
module.make_check_time_path(
{'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'repository', '5678'
)
== '/home/user/.borgmatic/checks/1234/repository'
)
def test_read_check_time_does_not_raise():
@ -395,14 +462,135 @@ def test_read_check_time_on_missing_file_does_not_raise():
assert module.read_check_time('/path') is None
def test_probe_for_check_time_uses_first_of_multiple_check_times():
flexmock(module).should_receive('make_check_time_path').and_return(
'~/.borgmatic/checks/1234/archives/5678'
).and_return('~/.borgmatic/checks/1234/archives/all')
flexmock(module).should_receive('read_check_time').and_return(1).and_return(2)
assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1
def test_probe_for_check_time_deduplicates_identical_check_time_paths():
flexmock(module).should_receive('make_check_time_path').and_return(
'~/.borgmatic/checks/1234/archives/5678'
).and_return('~/.borgmatic/checks/1234/archives/5678')
flexmock(module).should_receive('read_check_time').and_return(1).once()
assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1
def test_probe_for_check_time_skips_none_check_time():
flexmock(module).should_receive('make_check_time_path').and_return(
'~/.borgmatic/checks/1234/archives/5678'
).and_return('~/.borgmatic/checks/1234/archives/all')
flexmock(module).should_receive('read_check_time').and_return(None).and_return(2)
assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2
def test_probe_for_check_time_uses_single_check_time():
flexmock(module).should_receive('make_check_time_path').and_return(
'~/.borgmatic/checks/1234/archives/5678'
).and_return('~/.borgmatic/checks/1234/archives/all')
flexmock(module).should_receive('read_check_time').and_return(1).and_return(None)
assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1
def test_probe_for_check_time_returns_none_when_no_check_time_found():
flexmock(module).should_receive('make_check_time_path').and_return(
'~/.borgmatic/checks/1234/archives/5678'
).and_return('~/.borgmatic/checks/1234/archives/all')
flexmock(module).should_receive('read_check_time').and_return(None).and_return(None)
assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) is None
def test_upgrade_check_times_renames_old_check_paths_to_all():
base_path = '~/.borgmatic/checks/1234'
flexmock(module).should_receive('make_check_time_path').with_args(
object, object, 'archives', 'all'
).and_return(f'{base_path}/archives/all')
flexmock(module).should_receive('make_check_time_path').with_args(
object, object, 'data', 'all'
).and_return(f'{base_path}/data/all')
flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
True
)
flexmock(module.os.path).should_receive('isfile').with_args(
f'{base_path}/archives.temp'
).and_return(False)
flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
False
)
flexmock(module.os.path).should_receive('isfile').with_args(
f'{base_path}/data.temp'
).and_return(False)
flexmock(module.os).should_receive('rename').with_args(
f'{base_path}/archives', f'{base_path}/archives.temp'
).once()
flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once()
flexmock(module.os).should_receive('rename').with_args(
f'{base_path}/archives.temp', f'{base_path}/archives/all'
).once()
module.upgrade_check_times(flexmock(), flexmock())
def test_upgrade_check_times_skips_missing_check_paths():
flexmock(module).should_receive('make_check_time_path').and_return(
'~/.borgmatic/checks/1234/archives/all'
)
flexmock(module.os.path).should_receive('isfile').and_return(False)
flexmock(module.os).should_receive('rename').never()
flexmock(module.os).should_receive('mkdir').never()
module.upgrade_check_times(flexmock(), flexmock())
def test_upgrade_check_times_renames_stale_temporary_check_path():
base_path = '~/.borgmatic/checks/1234'
flexmock(module).should_receive('make_check_time_path').with_args(
object, object, 'archives', 'all'
).and_return(f'{base_path}/archives/all')
flexmock(module).should_receive('make_check_time_path').with_args(
object, object, 'data', 'all'
).and_return(f'{base_path}/data/all')
flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
False
)
flexmock(module.os.path).should_receive('isfile').with_args(
f'{base_path}/archives.temp'
).and_return(True)
flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
False
)
flexmock(module.os.path).should_receive('isfile').with_args(
f'{base_path}/data.temp'
).and_return(False)
flexmock(module.os).should_receive('rename').with_args(
f'{base_path}/archives', f'{base_path}/archives.temp'
).and_raise(FileNotFoundError)
flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once()
flexmock(module.os).should_receive('rename').with_args(
f'{base_path}/archives.temp', f'{base_path}/archives/all'
).once()
module.upgrade_check_times(flexmock(), flexmock())
def test_check_archives_with_progress_calls_borg_with_progress_parameter():
checks = ('repository',)
consistency_config = {'check_last': None}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module).should_receive('make_check_flags').and_return(())
flexmock(module).should_receive('execute_command').never()
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -429,11 +617,14 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter():
def test_check_archives_with_repair_calls_borg_with_repair_parameter():
checks = ('repository',)
consistency_config = {'check_last': None}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module).should_receive('make_check_flags').and_return(())
flexmock(module).should_receive('execute_command').never()
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@ -469,18 +660,15 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter():
def test_check_archives_calls_borg_with_parameters(checks):
check_last = flexmock()
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('make_check_flags').with_args(
'1.2.3',
{},
checks,
check_last,
prefix=None,
).and_return(())
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
insert_execute_command_mock(('borg', 'check', 'repo'))
flexmock(module).should_receive('make_check_time_path')
@ -500,11 +688,14 @@ def test_check_archives_with_json_error_raises():
checks = ('archives',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"unexpected": {"id": "repo"}}'
)
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
with pytest.raises(ValueError):
module.check_archives(
@ -521,9 +712,12 @@ def test_check_archives_with_missing_json_keys_raises():
checks = ('archives',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return('{invalid JSON')
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
with pytest.raises(ValueError):
module.check_archives(
@ -540,11 +734,14 @@ def test_check_archives_with_extract_check_calls_extract_only():
checks = ('extract',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module).should_receive('make_check_flags').never()
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
@ -564,11 +761,14 @@ def test_check_archives_with_extract_check_calls_extract_only():
def test_check_archives_with_log_info_calls_borg_with_info_parameter():
checks = ('repository',)
consistency_config = {'check_last': None}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module).should_receive('make_check_flags').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
insert_logging_mock(logging.INFO)
@ -589,11 +789,14 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter():
def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
checks = ('repository',)
consistency_config = {'check_last': None}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module).should_receive('make_check_flags').and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
insert_logging_mock(logging.DEBUG)
@ -613,11 +816,14 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
def test_check_archives_without_any_checks_bails():
consistency_config = {'check_last': None}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(())
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(())
insert_execute_command_never()
module.check_archives(
@ -634,18 +840,15 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
checks = ('repository',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('make_check_flags').with_args(
'1.2.3',
{},
checks,
check_last,
prefix=None,
).and_return(())
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
insert_execute_command_mock(('borg1', 'check', 'repo'))
flexmock(module).should_receive('make_check_time_path')
@ -666,18 +869,15 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
checks = ('repository',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('make_check_flags').with_args(
'1.2.3',
{},
checks,
check_last,
prefix=None,
).and_return(())
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('make_archive_filter_flags').and_return(())
flexmock(module).should_receive('make_archives_check_id').and_return(None)
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(())
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo'))
flexmock(module).should_receive('make_check_time_path')
@ -699,18 +899,15 @@ def test_check_archives_with_log_json_calls_borg_with_log_json_parameters():
check_last = flexmock()
storage_config = {}
consistency_config = {'check_last': check_last}
flexmock(module).should_receive('parse_checks')
flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks)
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
flexmock(module).should_receive('make_check_flags').with_args(