borgmatic-collective/borgmatic
borgmatic-collective/borgmatic
10
66
Fork 30
Code Issues 45 Pull Requests 5 Actions Packages Releases 145 Activity

Remove sections (#721).
Some checks failed
continuous-integration/drone/push Build is failing
Details

Browse Source
This commit is contained in:
Dan Helfman 2023-07-08 23:14:30 -07:00
parent 9cafc16052
commit 48b6a1679a
103 changed files with 2817 additions and 3050 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
NEWS
View File

@ -8,6 +8,13 @@
* #720: Fix an error when dumping a MySQL database and the "exclude_nodump" option is set.
* When merging two configuration files, error gracefully if the two files do not adhere to the same
format.
* #721: Remove configuration sections ("location:", "storage:", "hooks:" etc.), while still keeping
deprecated support for them. Now, all options are at the same level, and you don't need to worry
about commenting/uncommenting section headers when you change an option.
* #721: BREAKING: The retention prefix and the consistency prefix can no longer have different
values (unless one is not set).
* #721: BREAKING: The storage umask and the hooks umask can no longer have different values (unless
one is not set).
* BREAKING: Remove the deprecated (and silently ignored) "--successful" flag on the "list" action,
as newer versions of Borg list successful (non-checkpoint) archives by default.
* All deprecated configuration option values now generate warning logs.

6
borgmatic/actions/borg.py
View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def run_borg(
repository,
storage,
config,
local_borg_version,
borg_arguments,
global_arguments,
@ -28,7 +28,7 @@ def run_borg(
archive_name = borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
borg_arguments.archive,
storage,
config,
local_borg_version,
global_arguments,
local_path,
@ -36,7 +36,7 @@ def run_borg(
)
borgmatic.borg.borg.run_arbitrary_borg(
repository['path'],
storage,
config,
local_borg_version,
options=borg_arguments.options,
archive=archive_name,

4
borgmatic/actions/break_lock.py
View File

@ -8,7 +8,7 @@ logger = logging.getLogger(__name__)
def run_break_lock(
repository,
storage,
config,
local_borg_version,
break_lock_arguments,
global_arguments,
@ -26,7 +26,7 @@ def run_break_lock(
)
borgmatic.borg.break_lock.break_lock(
repository['path'],
storage,
config,
local_borg_version,
global_arguments,
local_path=local_path,

17
borgmatic/actions/check.py
View File

@ -10,10 +10,7 @@ logger = logging.getLogger(__name__)
def run_check(
config_filename,
repository,
location,
storage,
consistency,
hooks,
config,
hook_context,
local_borg_version,
check_arguments,
@ -30,8 +27,8 @@ def run_check(
return
borgmatic.hooks.command.execute_hook(
hooks.get('before_check'),
hooks.get('umask'),
config.get('before_check'),
config.get('umask'),
config_filename,
'pre-check',
global_arguments.dry_run,
@ -40,9 +37,7 @@ def run_check(
logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks')
borgmatic.borg.check.check_archives(
repository['path'],
location,
storage,
consistency,
config,
local_borg_version,
global_arguments,
local_path=local_path,
@ -53,8 +48,8 @@ def run_check(
force=check_arguments.force,
)
borgmatic.hooks.command.execute_hook(
hooks.get('after_check'),
hooks.get('umask'),
config.get('after_check'),
config.get('umask'),
config_filename,
'post-check',
global_arguments.dry_run,

14
borgmatic/actions/compact.py
View File

@ -11,9 +11,7 @@ logger = logging.getLogger(__name__)
def run_compact(
config_filename,
repository,
storage,
retention,
hooks,
config,
hook_context,
local_borg_version,
compact_arguments,
@ -31,8 +29,8 @@ def run_compact(
return
borgmatic.hooks.command.execute_hook(
hooks.get('before_compact'),
hooks.get('umask'),
config.get('before_compact'),
config.get('umask'),
config_filename,
'pre-compact',
global_arguments.dry_run,
@ -45,7 +43,7 @@ def run_compact(
borgmatic.borg.compact.compact_segments(
global_arguments.dry_run,
repository['path'],
storage,
config,
local_borg_version,
global_arguments,
local_path=local_path,
@ -59,8 +57,8 @@ def run_compact(
f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)'
)
borgmatic.hooks.command.execute_hook(
hooks.get('after_compact'),
hooks.get('umask'),
config.get('after_compact'),
config.get('umask'),
config_filename,
'post-compact',
global_arguments.dry_run,

2
borgmatic/actions/config/generate.py
View File

@ -2,6 +2,7 @@ import logging
import borgmatic.config.generate
import borgmatic.config.validate
import borgmatic.logger
logger = logging.getLogger(__name__)
@ -14,6 +15,7 @@ def run_generate(generate_arguments, global_arguments):
Raise FileExistsError if a file already exists at the destination path and the generate
arguments do not have overwrite set.
'''
borgmatic.logger.add_custom_log_levels()
dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
logger.answer(

3
borgmatic/actions/config/validate.py
View File

@ -1,6 +1,7 @@
import logging
import borgmatic.config.generate
import borgmatic.logger
logger = logging.getLogger(__name__)
@ -14,6 +15,8 @@ def run_validate(validate_arguments, configs):
loading machinery prior to here, so this function mainly exists to support additional validate
flags like "--show".
'''
borgmatic.logger.add_custom_log_levels()
if validate_arguments.show:
for config_path, config in configs.items():
if len(configs) > 1:

32
borgmatic/actions/create.py
View File

@ -17,7 +17,7 @@ import borgmatic.hooks.dump
logger = logging.getLogger(__name__)
def create_borgmatic_manifest(location, config_paths, dry_run):
def create_borgmatic_manifest(config, config_paths, dry_run):
'''
Create a borgmatic manifest file to store the paths to the configuration files used to create
the archive.
@ -25,7 +25,7 @@ def create_borgmatic_manifest(location, config_paths, dry_run):
if dry_run:
return
borgmatic_source_directory = location.get(
borgmatic_source_directory = config.get(
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
)
@ -49,9 +49,7 @@ def create_borgmatic_manifest(location, config_paths, dry_run):
def run_create(
config_filename,
repository,
location,
storage,
hooks,
config,
hook_context,
local_borg_version,
create_arguments,
@ -71,8 +69,8 @@ def run_create(
return
borgmatic.hooks.command.execute_hook(
hooks.get('before_backup'),
hooks.get('umask'),
config.get('before_backup'),
config.get('umask'),
config_filename,
'pre-backup',
global_arguments.dry_run,
@ -81,30 +79,25 @@ def run_create(
logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}')
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
'remove_database_dumps',
hooks,
config,
repository['path'],
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
active_dumps = borgmatic.hooks.dispatch.call_hooks(
'dump_databases',
hooks,
config,
repository['path'],
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
create_borgmatic_manifest(
location, global_arguments.used_config_paths, global_arguments.dry_run
)
create_borgmatic_manifest(config, global_arguments.used_config_paths, global_arguments.dry_run)
stream_processes = [process for processes in active_dumps.values() for process in processes]
json_output = borgmatic.borg.create.create_archive(
global_arguments.dry_run,
repository['path'],
location,
storage,
config,
local_borg_version,
global_arguments,
local_path=local_path,
@ -120,15 +113,14 @@ def run_create(
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
'remove_database_dumps',
hooks,
config,
config_filename,
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
borgmatic.hooks.command.execute_hook(
hooks.get('after_backup'),
hooks.get('umask'),
config.get('after_backup'),
config.get('umask'),
config_filename,
'post-backup',
global_arguments.dry_run,

6
borgmatic/actions/export_tar.py
View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def run_export_tar(
repository,
storage,
config,
local_borg_version,
export_tar_arguments,
global_arguments,
@ -31,7 +31,7 @@ def run_export_tar(
borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
export_tar_arguments.archive,
storage,
config,
local_borg_version,
global_arguments,
local_path,
@ -39,7 +39,7 @@ def run_export_tar(
),
export_tar_arguments.paths,
export_tar_arguments.destination,
storage,
config,
local_borg_version,
global_arguments,
local_path=local_path,

17
borgmatic/actions/extract.py
View File

@ -11,9 +11,7 @@ logger = logging.getLogger(__name__)
def run_extract(
config_filename,
repository,
location,
storage,
hooks,
config,
hook_context,
local_borg_version,
extract_arguments,
@ -25,8 +23,8 @@ def run_extract(
Run the "extract" action for the given repository.
'''
borgmatic.hooks.command.execute_hook(
hooks.get('before_extract'),
hooks.get('umask'),
config.get('before_extract'),
config.get('umask'),
config_filename,
'pre-extract',
global_arguments.dry_run,
@ -44,15 +42,14 @@ def run_extract(
borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
extract_arguments.archive,
storage,
config,
local_borg_version,
global_arguments,
local_path,
remote_path,
),
extract_arguments.paths,
location,
storage,
config,
local_borg_version,
global_arguments,
local_path=local_path,
@ -62,8 +59,8 @@ def run_extract(
progress=extract_arguments.progress,
)
borgmatic.hooks.command.execute_hook(
hooks.get('after_extract'),
hooks.get('umask'),
config.get('after_extract'),
config.get('umask'),
config_filename,
'post-extract',
global_arguments.dry_run,

6
borgmatic/actions/info.py
View File

@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
def run_info(
repository,
storage,
config,
local_borg_version,
info_arguments,
global_arguments,
@ -33,7 +33,7 @@ def run_info(
archive_name = borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
info_arguments.archive,
storage,
config,
local_borg_version,
global_arguments,
local_path,
@ -41,7 +41,7 @@ def run_info(
)
json_output = borgmatic.borg.info.display_archives_info(
repository['path'],
storage,
config,
local_borg_version,
borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
global_arguments,

6
borgmatic/actions/list.py
View File

@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
def run_list(
repository,
storage,
config,
local_borg_version,
list_arguments,
global_arguments,
@ -34,7 +34,7 @@ def run_list(
archive_name = borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
list_arguments.archive,
storage,
config,
local_borg_version,
global_arguments,
local_path,
@ -42,7 +42,7 @@ def run_list(
)
json_output = borgmatic.borg.list.list_archive(
repository['path'],
storage,
config,
local_borg_version,
borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
global_arguments,

6
borgmatic/actions/mount.py
View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def run_mount(
repository,
storage,
config,
local_borg_version,
mount_arguments,
global_arguments,
@ -34,14 +34,14 @@ def run_mount(
borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
mount_arguments.archive,
storage,
config,
local_borg_version,
global_arguments,
local_path,
remote_path,
),
mount_arguments,
storage,
config,
local_borg_version,
global_arguments,
local_path=local_path,

15
borgmatic/actions/prune.py
View File

@ -10,9 +10,7 @@ logger = logging.getLogger(__name__)
def run_prune(
config_filename,
repository,
storage,
retention,
hooks,
config,
hook_context,
local_borg_version,
prune_arguments,
@ -30,8 +28,8 @@ def run_prune(
return
borgmatic.hooks.command.execute_hook(
hooks.get('before_prune'),
hooks.get('umask'),
config.get('before_prune'),
config.get('umask'),
config_filename,
'pre-prune',
global_arguments.dry_run,
@ -41,8 +39,7 @@ def run_prune(
borgmatic.borg.prune.prune_archives(
global_arguments.dry_run,
repository['path'],
storage,
retention,
config,
local_borg_version,
prune_arguments,
global_arguments,
@ -50,8 +47,8 @@ def run_prune(
remote_path=remote_path,
)
borgmatic.hooks.command.execute_hook(
hooks.get('after_prune'),
hooks.get('umask'),
config.get('after_prune'),
config.get('umask'),
config_filename,
'post-prune',
global_arguments.dry_run,

4
borgmatic/actions/rcreate.py
View File

@ -8,7 +8,7 @@ logger = logging.getLogger(__name__)
def run_rcreate(
repository,
storage,
config,
local_borg_version,
rcreate_arguments,
global_arguments,
@ -27,7 +27,7 @@ def run_rcreate(
borgmatic.borg.rcreate.create_repository(
global_arguments.dry_run,
repository['path'],
storage,
config,
local_borg_version,
global_arguments,
rcreate_arguments.encryption_mode,

74
borgmatic/actions/restore.py
View File

@ -18,12 +18,12 @@ UNSPECIFIED_HOOK = object()
def get_configured_database(
hooks, archive_database_names, hook_name, database_name, configuration_database_name=None
config, archive_database_names, hook_name, database_name, configuration_database_name=None
):
'''
Find the first database with the given hook name and database name in the configured hooks
dict and the given archive database names dict (from hook name to database names contained in
a particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all database
Find the first database with the given hook name and database name in the configuration dict and
the given archive database names dict (from hook name to database names contained in a
particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all database
hooks for the named database. If a configuration database name is given, use that instead of the
database name to lookup the database in the given hooks configuration.
@ -33,9 +33,13 @@ def get_configured_database(
configuration_database_name = database_name
if hook_name == UNSPECIFIED_HOOK:
hooks_to_search = hooks
hooks_to_search = {
hook_name: value
for (hook_name, value) in config.items()
if hook_name in borgmatic.hooks.dump.DATABASE_HOOK_NAMES
}
else:
hooks_to_search = {hook_name: hooks[hook_name]}
hooks_to_search = {hook_name: config[hook_name]}
return next(
(
@ -58,9 +62,7 @@ def get_configured_hook_name_and_database(hooks, database_name):
def restore_single_database(
repository,
location,
storage,
hooks,
config,
local_borg_version,
global_arguments,
local_path,
@ -81,10 +83,9 @@ def restore_single_database(
dump_pattern = borgmatic.hooks.dispatch.call_hooks(
'make_database_dump_pattern',
hooks,
config,
repository['path'],
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
database['name'],
)[hook_name]
@ -94,8 +95,7 @@ def restore_single_database(
repository=repository['path'],
archive=archive_name,
paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
location_config=location,
storage_config=storage,
config=config,
local_borg_version=local_borg_version,
global_arguments=global_arguments,
local_path=local_path,
@ -112,7 +112,7 @@ def restore_single_database(
{hook_name: [database]},
repository['path'],
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
config,
global_arguments.dry_run,
extract_process,
connection_params,
@ -122,21 +122,20 @@ def restore_single_database(
def collect_archive_database_names(
repository,
archive,
location,
storage,
config,
local_borg_version,
global_arguments,
local_path,
remote_path,
):
'''
Given a local or remote repository path, a resolved archive name, a location configuration dict,
a storage configuration dict, the local Borg version, global_arguments an argparse.Namespace,
and local and remote Borg paths, query the archive for the names of databases it contains and
return them as a dict from hook name to a sequence of database names.
Given a local or remote repository path, a resolved archive name, a configuration dict, the
local Borg version, global_arguments an argparse.Namespace, and local and remote Borg paths,
query the archive for the names of databases it contains and return them as a dict from hook
name to a sequence of database names.
'''
borgmatic_source_directory = os.path.expanduser(
location.get(
config.get(
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
)
).lstrip('/')
@ -146,7 +145,7 @@ def collect_archive_database_names(
dump_paths = borgmatic.borg.list.capture_archive_listing(
repository,
archive,
storage,
config,
local_borg_version,
global_arguments,
list_path=parent_dump_path,
@ -249,9 +248,7 @@ def ensure_databases_found(restore_names, remaining_restore_names, found_names):
def run_restore(
repository,
location,
storage,
hooks,
config,
local_borg_version,
restore_arguments,
global_arguments,
@ -275,17 +272,16 @@ def run_restore(
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
'remove_database_dumps',
hooks,
config,
repository['path'],
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)
archive_name = borgmatic.borg.rlist.resolve_archive_name(
repository['path'],
restore_arguments.archive,
storage,
config,
local_borg_version,
global_arguments,
local_path,
@ -294,8 +290,7 @@ def run_restore(
archive_database_names = collect_archive_database_names(
repository['path'],
archive_name,
location,
storage,
config,
local_borg_version,
global_arguments,
local_path,
@ -315,7 +310,7 @@ def run_restore(
for hook_name, database_names in restore_names.items():
for database_name in database_names:
found_hook_name, found_database = get_configured_database(
hooks, archive_database_names, hook_name, database_name
config, archive_database_names, hook_name, database_name
)
if not found_database:
@ -327,9 +322,7 @@ def run_restore(
found_names.add(database_name)
restore_single_database(
repository,
location,
storage,
hooks,
config,
local_borg_version,
global_arguments,
local_path,
@ -340,12 +333,12 @@ def run_restore(
connection_params,
)
# For any database that weren't found via exact matches in the hooks configuration, try to
# fallback to "all" entries.
# For any database that weren't found via exact matches in the configuration, try to fallback
# to "all" entries.
for hook_name, database_names in remaining_restore_names.items():
for database_name in database_names:
found_hook_name, found_database = get_configured_database(
hooks, archive_database_names, hook_name, database_name, 'all'
config, archive_database_names, hook_name, database_name, 'all'
)
if not found_database:
@ -357,9 +350,7 @@ def run_restore(
restore_single_database(
repository,
location,
storage,
hooks,
config,
local_borg_version,
global_arguments,
local_path,
@ -372,10 +363,9 @@ def run_restore(
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
'remove_database_dumps',
hooks,
config,
repository['path'],
borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location,
global_arguments.dry_run,
)

4
borgmatic/actions/rinfo.py
View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def run_rinfo(
repository,
storage,
config,
local_borg_version,
rinfo_arguments,
global_arguments,
@ -31,7 +31,7 @@ def run_rinfo(
json_output = borgmatic.borg.rinfo.display_repository_info(
repository['path'],
storage,
config,
local_borg_version,
rinfo_arguments=rinfo_arguments,
global_arguments=global_arguments,

4
borgmatic/actions/rlist.py
View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def run_rlist(
repository,
storage,
config,
local_borg_version,
rlist_arguments,
global_arguments,
@ -29,7 +29,7 @@ def run_rlist(
json_output = borgmatic.borg.rlist.list_repository(
repository['path'],
storage,
config,
local_borg_version,
rlist_arguments=rlist_arguments,
global_arguments=global_arguments,

4
borgmatic/actions/transfer.py
View File

@ -7,7 +7,7 @@ logger = logging.getLogger(__name__)
def run_transfer(
repository,
storage,
config,
local_borg_version,
transfer_arguments,
global_arguments,
@ -23,7 +23,7 @@ def run_transfer(
borgmatic.borg.transfer.transfer_archives(
global_arguments.dry_run,
repository['path'],
storage,
config,
local_borg_version,
transfer_arguments,
global_arguments,

8
borgmatic/borg/borg.py
View File

@ -13,7 +13,7 @@ BORG_SUBCOMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'}
def run_arbitrary_borg(
repository_path,
storage_config,
config,
local_borg_version,
options,
archive=None,
@ -21,13 +21,13 @@ def run_arbitrary_borg(
remote_path=None,
):
'''
Given a local or remote repository path, a storage config dict, the local Borg version, a
Given a local or remote repository path, a configuration dict, the local Borg version, a
sequence of arbitrary command-line Borg options, and an optional archive name, run an arbitrary
Borg command, passing in REPOSITORY and ARCHIVE environment variables for optional use in the
command.
'''
borgmatic.logger.add_custom_log_levels()
lock_wait = storage_config.get('lock_wait', None)
lock_wait = config.get('lock_wait', None)
try:
options = options[1:] if options[0] == '--' else options
@ -61,7 +61,7 @@ def run_arbitrary_borg(
borg_local_path=local_path,
shell=True,
extra_environment=dict(
(environment.make_environment(storage_config) or {}),
(environment.make_environment(config) or {}),
**{
'BORG_REPO': repository_path,
'ARCHIVE': archive if archive else '',

12
borgmatic/borg/break_lock.py
View File

@ -8,19 +8,19 @@ logger = logging.getLogger(__name__)
def break_lock(
repository_path,
storage_config,
config,
local_borg_version,
global_arguments,
local_path='borg',
remote_path=None,
):
'''
Given a local or remote repository path, a storage configuration dict, the local Borg version,
an argparse.Namespace of global arguments, and optional local and remote Borg paths, break any
Given a local or remote repository path, a configuration dict, the local Borg version, an
argparse.Namespace of global arguments, and optional local and remote Borg paths, break any
repository and cache locks leftover from Borg aborting.
'''
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None)
full_command = (
(local_path, 'break-lock')
@ -33,5 +33,5 @@ def break_lock(
+ flags.make_repository_flags(repository_path, local_borg_version)
)
borg_environment = environment.make_environment(storage_config)
borg_environment = environment.make_environment(config)
execute_command(full_command, borg_local_path=local_path, extra_environment=borg_environment)

116
borgmatic/borg/check.py
View File

@ -19,12 +19,12 @@ DEFAULT_CHECKS = (
logger = logging.getLogger(__name__)
def parse_checks(consistency_config, only_checks=None):
def parse_checks(config, only_checks=None):
'''
Given a consistency config with a "checks" sequence of dicts and an optional list of override
Given a configuration dict with a "checks" sequence of dicts and an optional list of override
checks, return a tuple of named checks to run.
For example, given a retention config of:
For example, given a config of:
{'checks': ({'name': 'repository'}, {'name': 'archives'})}
@ -36,8 +36,7 @@ def parse_checks(consistency_config, only_checks=None):
has a name of "disabled", return an empty tuple, meaning that no checks should be run.
'''
checks = only_checks or tuple(
check_config['name']
for check_config in (consistency_config.get('checks', None) or DEFAULT_CHECKS)
check_config['name'] for check_config in (config.get('checks', None) or DEFAULT_CHECKS)
)
checks = tuple(check.lower() for check in checks)
if 'disabled' in checks:
@ -90,23 +89,22 @@ def parse_frequency(frequency):
def filter_checks_on_frequency(
location_config,
consistency_config,
config,
borg_repository_id,
checks,
force,
archives_check_id=None,
):
'''
Given a location config, a consistency config with a "checks" sequence of dicts, a Borg
repository ID, a sequence of checks, whether to force checks to run, and an ID for the archives
check potentially being run (if any), filter down those checks based on the configured
"frequency" for each check as compared to its check time file.
Given a configuration dict with a "checks" sequence of dicts, a Borg repository ID, a sequence
of checks, whether to force checks to run, and an ID for the archives check potentially being
run (if any), filter down those checks based on the configured "frequency" for each check as
compared to its check time file.
In other words, a check whose check time file's timestamp is too new (based on the configured
frequency) will get cut from the returned sequence of checks. Example:
consistency_config = {
config = {
'checks': [
{
'name': 'archives',
@ -115,9 +113,9 @@ def filter_checks_on_frequency(
]
}
When this function is called with that consistency_config and "archives" in checks, "archives"
will get filtered out of the returned result if its check time file is newer than 2 weeks old,
indicating that it's not yet time to run that check again.
When this function is called with that config and "archives" in checks, "archives" will get
filtered out of the returned result if its check time file is newer than 2 weeks old, indicating
that it's not yet time to run that check again.
Raise ValueError if a frequency cannot be parsed.
'''
@ -126,7 +124,7 @@ def filter_checks_on_frequency(
if force:
return tuple(filtered_checks)
for check_config in consistency_config.get('checks', DEFAULT_CHECKS):
for check_config in config.get('checks', DEFAULT_CHECKS):
check = check_config['name']
if checks and check not in checks:
continue
@ -135,9 +133,7 @@ def filter_checks_on_frequency(
if not frequency_delta:
continue
check_time = probe_for_check_time(
location_config, borg_repository_id, check, archives_check_id
)
check_time = probe_for_check_time(config, borg_repository_id, check, archives_check_id)
if not check_time:
continue
@ -153,13 +149,11 @@ def filter_checks_on_frequency(
return tuple(filtered_checks)
def make_archive_filter_flags(
local_borg_version, storage_config, checks, check_last=None, prefix=None
):
def make_archive_filter_flags(local_borg_version, config, checks, check_last=None, prefix=None):
'''
Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the
check last value, and a consistency check prefix, transform the checks into tuple of
command-line flags for filtering archives in a check command.
Given the local Borg version, a configuration dict, a parsed sequence of checks, the check last
value, and a consistency check prefix, transform the checks into tuple of command-line flags for
filtering archives in a check command.
If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if
a prefix value is given and "archives" is in checks, then include a "--match-archives" flag.
@ -174,8 +168,8 @@ def make_archive_filter_flags(
if prefix
else (
flags.make_match_archives_flags(
storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
config.get('match_archives'),
config.get('archive_name_format'),
local_borg_version,
)
)
@ -237,14 +231,14 @@ def make_check_flags(checks, archive_filter_flags):
)
def make_check_time_path(location_config, borg_repository_id, check_type, archives_check_id=None):
def make_check_time_path(config, borg_repository_id, check_type, archives_check_id=None):
'''
Given a location configuration dict, a Borg repository ID, the name of a check type
("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
path for recording that check's time (the time of that check last occurring).
Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
"archives", etc.), and a unique hash of the archives filter flags, return a path for recording
that check's time (the time of that check last occurring).
'''
borgmatic_source_directory = os.path.expanduser(
location_config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY)
config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY)
)
if check_type in ('archives', 'data'):
@ -287,11 +281,11 @@ def read_check_time(path):
return None
def probe_for_check_time(location_config, borg_repository_id, check, archives_check_id):
def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
'''
Given a location configuration dict, a Borg repository ID, the name of a check type
("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
the corresponding check time or None if such a check time does not exist.
Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
"archives", etc.), and a unique hash of the archives filter flags, return a the corresponding
check time or None if such a check time does not exist.
When the check type is "archives" or "data", this function probes two different paths to find
the check time, e.g.:
@ -311,8 +305,8 @@ def probe_for_check_time(location_config, borg_repository_id, check, archives_ch
read_check_time(group[0])
for group in itertools.groupby(
(
make_check_time_path(location_config, borg_repository_id, check, archives_check_id),
make_check_time_path(location_config, borg_repository_id, check),
make_check_time_path(config, borg_repository_id, check, archives_check_id),
make_check_time_path(config, borg_repository_id, check),
)
)
)
@ -323,10 +317,10 @@ def probe_for_check_time(location_config, borg_repository_id, check, archives_ch
return None
def upgrade_check_times(location_config, borg_repository_id):
def upgrade_check_times(config, borg_repository_id):
'''
Given a location configuration dict and a Borg repository ID, upgrade any corresponding check
times on disk from old-style paths to new-style paths.
Given a configuration dict and a Borg repository ID, upgrade any corresponding check times on
disk from old-style paths to new-style paths.
Currently, the only upgrade performed is renaming an archive or data check path that looks like:
@ -337,7 +331,7 @@ def upgrade_check_times(location_config, borg_repository_id):
~/.borgmatic/checks/1234567890/archives/all
'''
for check_type in ('archives', 'data'):
new_path = make_check_time_path(location_config, borg_repository_id, check_type, 'all')
new_path = make_check_time_path(config, borg_repository_id, check_type, 'all')
old_path = os.path.dirname(new_path)
temporary_path = f'{old_path}.temp'
@ -357,9 +351,7 @@ def upgrade_check_times(location_config, borg_repository_id):
def check_archives(
repository_path,
location_config,
storage_config,
consistency_config,
config,
local_borg_version,
global_arguments,
local_path='borg',
@ -370,10 +362,9 @@ def check_archives(
force=None,
):
'''
Given a local or remote repository path, a storage config dict, a consistency config dict,
local/remote commands to run, whether to include progress information, whether to attempt a
repair, and an optional list of checks to use instead of configured checks, check the contained
Borg archives for consistency.
Given a local or remote repository path, a configuration dict, local/remote commands to run,
whether to include progress information, whether to attempt a repair, and an optional list of
checks to use instead of configured checks, check the contained Borg archives for consistency.
If there are no consistency checks to run, skip running them.
@ -383,7 +374,7 @@ def check_archives(
borg_repository_id = json.loads(
rinfo.display_repository_info(
repository_path,
storage_config,
config,
local_borg_version,
argparse.Namespace(json=True),
global_arguments,
@ -394,21 +385,20 @@ def check_archives(
except (json.JSONDecodeError, KeyError):
raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
upgrade_check_times(location_config, borg_repository_id)
upgrade_check_times(config, borg_repository_id)
check_last = consistency_config.get('check_last', None)
prefix = consistency_config.get('prefix')
configured_checks = parse_checks(consistency_config, only_checks)
check_last = config.get('check_last', None)
prefix = config.get('prefix')
configured_checks = parse_checks(config, only_checks)
lock_wait = None
extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
archive_filter_flags = make_archive_filter_flags(
local_borg_version, storage_config, configured_checks, check_last, prefix
local_borg_version, config, configured_checks, check_last, prefix
)
archives_check_id = make_archives_check_id(archive_filter_flags)
checks = filter_checks_on_frequency(
location_config,
consistency_config,
config,
borg_repository_id,
configured_checks,
force,
@ -416,7 +406,7 @@ def check_archives(
)
if set(checks).intersection({'repository', 'archives', 'data'}):
lock_wait = storage_config.get('lock_wait')
lock_wait = config.get('lock_wait')
verbosity_flags = ()
if logger.isEnabledFor(logging.INFO):
@ -437,7 +427,7 @@ def check_archives(
+ flags.make_repository_flags(repository_path, local_borg_version)
)
borg_environment = environment.make_environment(storage_config)
borg_environment = environment.make_environment(config)
# The Borg repair option triggers an interactive prompt, which won't work when output is
# captured. And progress messes with the terminal directly.
@ -450,12 +440,12 @@ def check_archives(
for check in checks:
write_check_time(
make_check_time_path(location_config, borg_repository_id, check, archives_check_id)
make_check_time_path(config, borg_repository_id, check, archives_check_id)
)
if 'extract' in checks:
extract.extract_last_archive_dry_run(
storage_config,
config,
local_borg_version,
global_arguments,
repository_path,
@ -463,4 +453,4 @@ def check_archives(
local_path,
remote_path,
)
write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract'))
write_check_time(make_check_time_path(config, borg_repository_id, 'extract'))

14
borgmatic/borg/compact.py
View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def compact_segments(
dry_run,
repository_path,
storage_config,
config,
local_borg_version,
global_arguments,
local_path='borg',
@ -19,12 +19,12 @@ def compact_segments(
threshold=None,
):
'''
Given dry-run flag, a local or remote repository path, a storage config dict, and the local
Borg version, compact the segments in a repository.
Given dry-run flag, a local or remote repository path, a configuration dict, and the local Borg
version, compact the segments in a repository.
'''
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
extra_borg_options = storage_config.get('extra_borg_options', {}).get('compact', '')
umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None)
extra_borg_options = config.get('extra_borg_options', {}).get('compact', '')
full_command = (
(local_path, 'compact')
@ -49,5 +49,5 @@ def compact_segments(
full_command,
output_log_level=logging.INFO,
borg_local_path=local_path,
extra_environment=environment.make_environment(storage_config),
extra_environment=environment.make_environment(config),
)

111
borgmatic/borg/create.py
View File

@ -146,12 +146,12 @@ def ensure_files_readable(*filename_lists):
open(file_object).close()
def make_pattern_flags(location_config, pattern_filename=None):
def make_pattern_flags(config, pattern_filename=None):
'''
Given a location config dict with a potential patterns_from option, and a filename containing
any additional patterns, return the corresponding Borg flags for those files as a tuple.
Given a configuration dict with a potential patterns_from option, and a filename containing any
additional patterns, return the corresponding Borg flags for those files as a tuple.
'''
pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
pattern_filenames = tuple(config.get('patterns_from') or ()) + (
(pattern_filename,) if pattern_filename else ()
)
@ -162,12 +162,12 @@ def make_pattern_flags(location_config, pattern_filename=None):
)
def make_exclude_flags(location_config, exclude_filename=None):
def make_exclude_flags(config, exclude_filename=None):
'''
Given a location config dict with various exclude options, and a filename containing any exclude
Given a configuration dict with various exclude options, and a filename containing any exclude
patterns, return the corresponding Borg flags as a tuple.
'''
exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
exclude_filenames = tuple(config.get('exclude_from') or ()) + (
(exclude_filename,) if exclude_filename else ()
)
exclude_from_flags = tuple(
@ -175,17 +175,15 @@ def make_exclude_flags(location_config, exclude_filename=None):
('--exclude-from', exclude_filename) for exclude_filename in exclude_filenames
)
)
caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else ()
if_present_flags = tuple(
itertools.chain.from_iterable(
('--exclude-if-present', if_present)
for if_present in location_config.get('exclude_if_present', ())
for if_present in config.get('exclude_if_present', ())
)
)
keep_exclude_tags_flags = (
('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else ()
)
exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else ()
keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
return (
exclude_from_flags
@ -326,8 +324,7 @@ def check_all_source_directories_exist(source_directories):
def create_archive(
dry_run,
repository_path,
location_config,
storage_config,
config,
local_borg_version,
global_arguments,
local_path='borg',
@ -339,72 +336,70 @@ def create_archive(
stream_processes=None,
):
'''
Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
storage config dict, create a Borg archive and return Borg's JSON output (if any).
Given vebosity/dry-run flags, a local or remote repository path, and a configuration dict,
create a Borg archive and return Borg's JSON output (if any).
If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
create command while also triggering the given processes to produce output.
'''
borgmatic.logger.add_custom_log_levels()
borgmatic_source_directories = expand_directories(
collect_borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
collect_borgmatic_source_directories(config.get('borgmatic_source_directory'))
)
if location_config.get('source_directories_must_exist', False):
check_all_source_directories_exist(location_config.get('source_directories'))
if config.get('source_directories_must_exist', False):
check_all_source_directories_exist(config.get('source_directories'))
sources = deduplicate_directories(
map_directories_to_devices(
expand_directories(
tuple(location_config.get('source_directories', ()))
tuple(config.get('source_directories', ()))
+ borgmatic_source_directories
+ tuple(global_arguments.used_config_paths)
)
),
additional_directory_devices=map_directories_to_devices(
expand_directories(pattern_root_directories(location_config.get('patterns')))
expand_directories(pattern_root_directories(config.get('patterns')))
),
)
ensure_files_readable(location_config.get('patterns_from'), location_config.get('exclude_from'))
ensure_files_readable(config.get('patterns_from'), config.get('exclude_from'))
try:
working_directory = os.path.expanduser(location_config.get('working_directory'))
working_directory = os.path.expanduser(config.get('working_directory'))
except TypeError:
working_directory = None
pattern_file = (
write_pattern_file(location_config.get('patterns'), sources)
if location_config.get('patterns') or location_config.get('patterns_from')
write_pattern_file(config.get('patterns'), sources)
if config.get('patterns') or config.get('patterns_from')
else None
)
exclude_file = write_pattern_file(
expand_home_directories(location_config.get('exclude_patterns'))
)
checkpoint_interval = storage_config.get('checkpoint_interval', None)
checkpoint_volume = storage_config.get('checkpoint_volume', None)
chunker_params = storage_config.get('chunker_params', None)
compression = storage_config.get('compression', None)
upload_rate_limit = storage_config.get('upload_rate_limit', None)
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
exclude_file = write_pattern_file(expand_home_directories(config.get('exclude_patterns')))
checkpoint_interval = config.get('checkpoint_interval', None)
checkpoint_volume = config.get('checkpoint_volume', None)
chunker_params = config.get('chunker_params', None)
compression = config.get('compression', None)
upload_rate_limit = config.get('upload_rate_limit', None)
umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None)
list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
files_cache = location_config.get('files_cache')
archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
files_cache = config.get('files_cache')
archive_name_format = config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
extra_borg_options = config.get('extra_borg_options', {}).get('create', '')
if feature.available(feature.Feature.ATIME, local_borg_version):
atime_flags = ('--atime',) if location_config.get('atime') is True else ()
atime_flags = ('--atime',) if config.get('atime') is True else ()
else:
atime_flags = ('--noatime',) if location_config.get('atime') is False else ()
atime_flags = ('--noatime',) if config.get('atime') is False else ()
if feature.available(feature.Feature.NOFLAGS, local_borg_version):
noflags_flags = ('--noflags',) if location_config.get('flags') is False else ()
noflags_flags = ('--noflags',) if config.get('flags') is False else ()
else:
noflags_flags = ('--nobsdflags',) if location_config.get('flags') is False else ()
noflags_flags = ('--nobsdflags',) if config.get('flags') is False else ()
if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version):
numeric_ids_flags = ('--numeric-ids',) if location_config.get('numeric_ids') else ()
numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
else:
numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_ids') else ()
numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else ()
if feature.available(feature.Feature.UPLOAD_RATELIMIT, local_borg_version):
upload_ratelimit_flags = (
@ -415,7 +410,7 @@ def create_archive(
('--remote-ratelimit', str(upload_rate_limit)) if upload_rate_limit else ()
)
if stream_processes and location_config.get('read_special') is False:
if stream_processes and config.get('read_special') is False:
logger.warning(
f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
)
@ -423,23 +418,19 @@ def create_archive(
create_command = (
tuple(local_path.split(' '))
+ ('create',)
+ make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
+ make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
+ make_pattern_flags(config, pattern_file.name if pattern_file else None)
+ make_exclude_flags(config, exclude_file.name if exclude_file else None)
+ (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
+ (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ())
+ (('--chunker-params', chunker_params) if chunker_params else ())
+ (('--compression', compression) if compression else ())
+ upload_ratelimit_flags
+ (
('--one-file-system',)
if location_config.get('one_file_system') or stream_processes
else ()
)
+ (('--one-file-system',) if config.get('one_file_system') or stream_processes else ())
+ numeric_ids_flags
+ atime_flags
+ (('--noctime',) if location_config.get('ctime') is False else ())
+ (('--nobirthtime',) if location_config.get('birthtime') is False else ())
+ (('--read-special',) if location_config.get('read_special') or stream_processes else ())
+ (('--noctime',) if config.get('ctime') is False else ())
+ (('--nobirthtime',) if config.get('birthtime') is False else ())
+ (('--read-special',) if config.get('read_special') or stream_processes else ())
+ noflags_flags
+ (('--files-cache', files_cache) if files_cache else ())
+ (('--remote-path', remote_path) if remote_path else ())
@ -470,11 +461,11 @@ def create_archive(
# the terminal directly.
output_file = DO_NOT_CAPTURE if progress else None
borg_environment = environment.make_environment(storage_config)
borg_environment = environment.make_environment(config)
# If database hooks are enabled (as indicated by streaming processes), exclude files that might
# cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
if stream_processes and not location_config.get('read_special'):
if stream_processes and not config.get('read_special'):
logger.debug(f'{repository_path}: Collecting special file paths')
special_file_paths = collect_special_file_paths(
create_command,
@ -490,11 +481,11 @@ def create_archive(
)
exclude_file = write_pattern_file(
expand_home_directories(
tuple(location_config.get('exclude_patterns') or ()) + special_file_paths
tuple(config.get('exclude_patterns') or ()) + special_file_paths
),
pattern_file=exclude_file,
)
create_command += make_exclude_flags(location_config, exclude_file.name)
create_command += make_exclude_flags(config, exclude_file.name)
create_command += (
(('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())

8
borgmatic/borg/environment.py
View File

@ -17,15 +17,15 @@ DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
}
def make_environment(storage_config):
def make_environment(config):
'''
Given a borgmatic storage configuration dict, return its options converted to a Borg environment
Given a borgmatic configuration dict, return its options converted to a Borg environment
variable dict.
'''
environment = {}
for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
value = storage_config.get(option_name)
value = config.get(option_name)
if value:
environment[environment_variable_name] = str(value)
@ -34,7 +34,7 @@ def make_environment(storage_config):
option_name,
environment_variable_name,
) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
value = storage_config.get(option_name, False)
value = config.get(option_name, False)
environment[environment_variable_name] = 'yes' if value else 'no'
return environment

16
borgmatic/borg/export_tar.py
View File

@ -13,7 +13,7 @@ def export_tar_archive(
archive,
paths,
destination_path,
storage_config,
config,
local_borg_version,
global_arguments,
local_path='borg',
@ -24,16 +24,16 @@ def export_tar_archive(
):
'''
Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
export from the archive, a destination path to export to, a storage configuration dict, the
local Borg version, optional local and remote Borg paths, an optional filter program, whether to
include per-file details, and an optional number of path components to strip, export the archive
into the given destination path as a tar-formatted file.
export from the archive, a destination path to export to, a configuration dict, the local Borg
version, optional local and remote Borg paths, an optional filter program, whether to include
per-file details, and an optional number of path components to strip, export the archive into
the given destination path as a tar-formatted file.
If the destination path is "-", then stream the output to stdout instead of to a file.
'''
borgmatic.logger.add_custom_log_levels()
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None)
full_command = (
(local_path, 'export-tar')
@ -70,5 +70,5 @@ def export_tar_archive(
output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
output_log_level=output_log_level,
borg_local_path=local_path,
extra_environment=environment.make_environment(storage_config),
extra_environment=environment.make_environment(config),
)

23
borgmatic/borg/extract.py
View File

@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
def extract_last_archive_dry_run(
storage_config,
config,
local_borg_version,
global_arguments,
repository_path,
@ -32,7 +32,7 @@ def extract_last_archive_dry_run(
last_archive_name = rlist.resolve_archive_name(
repository_path,
'latest',
storage_config,
config,
local_borg_version,
global_arguments,
local_path,
@ -43,7 +43,7 @@ def extract_last_archive_dry_run(
return
list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
borg_environment = environment.make_environment(storage_config)
borg_environment = environment.make_environment(config)
full_extract_command = (
(local_path, 'extract', '--dry-run')
+ (('--remote-path', remote_path) if remote_path else ())
@ -66,8 +66,7 @@ def extract_archive(
repository,
archive,
paths,
location_config,
storage_config,
config,
local_borg_version,
global_arguments,
local_path='borg',
@ -80,22 +79,22 @@ def extract_archive(
'''
Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
restore from the archive, the local Borg version string, an argparse.Namespace of global
arguments, location/storage configuration dicts, optional local and remote Borg paths, and an
optional destination path to extract to, extract the archive into the current directory.
arguments, a configuration dict, optional local and remote Borg paths, and an optional
destination path to extract to, extract the archive into the current directory.
If extract to stdout is True, then start the extraction streaming to stdout, and return that
extract process as an instance of subprocess.Popen.
'''
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None)
if progress and extract_to_stdout:
raise ValueError('progress and extract_to_stdout cannot both be set')
if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version):
numeric_ids_flags = ('--numeric-ids',) if location_config.get('numeric_ids') else ()
numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
else:
numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_ids') else ()
numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else ()
if strip_components == 'all':
if not paths:
@ -127,7 +126,7 @@ def extract_archive(
+ (tuple(paths) if paths else ())
)
borg_environment = environment.make_environment(storage_config)
borg_environment = environment.make_environment(config)
# The progress output isn't compatible with captured and logged output, as progress messes with
# the terminal directly.

14
borgmatic/borg/info.py
View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def display_archives_info(
repository_path,
storage_config,
config,
local_borg_version,
info_arguments,
global_arguments,
@ -17,12 +17,12 @@ def display_archives_info(
remote_path=None,
):
'''
Given a local or remote repository path, a storage config dict, the local Borg version, global
Given a local or remote repository path, a configuration dict, the local Borg version, global
arguments as an argparse.Namespace, and the arguments to the info action, display summary
information for Borg archives in the repository or return JSON summary information.
'''
borgmatic.logger.add_custom_log_levels()
lock_wait = storage_config.get('lock_wait', None)
lock_wait = config.get('lock_wait', None)
full_command = (
(local_path, 'info')
@ -50,8 +50,8 @@ def display_archives_info(
flags.make_match_archives_flags(
info_arguments.match_archives
or info_arguments.archive
or storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
or config.get('match_archives'),
config.get('archive_name_format'),
local_borg_version,
)
)
@ -65,12 +65,12 @@ def display_archives_info(
if info_arguments.json:
return execute_command_and_capture_output(
full_command,
extra_environment=environment.make_environment(storage_config),
extra_environment=environment.make_environment(config),
)
else:
execute_command(
full_command,
output_log_level=logging.ANSWER,
borg_local_path=local_path,
extra_environment=environment.make_environment(storage_config),
extra_environment=environment.make_environment(config),
)

30
borgmatic/borg/list.py
View File

@ -21,7 +21,7 @@ MAKE_FLAGS_EXCLUDES = (
def make_list_command(
repository_path,
storage_config,
config,
local_borg_version,
list_arguments,
global_arguments,
@ -29,11 +29,11 @@ def make_list_command(
remote_path=None,
):
'''
Given a local or remote repository path, a storage config dict, the arguments to the list
action, and local and remote Borg paths, return a command as a tuple to list archives or paths
within an archive.
Given a local or remote repository path, a configuration dict, the arguments to the list action,
and local and remote Borg paths, return a command as a tuple to list archives or paths within an
archive.
'''
lock_wait = storage_config.get('lock_wait', None)
lock_wait = config.get('lock_wait', None)
return (
(local_path, 'list')
@ -89,7 +89,7 @@ def make_find_paths(find_paths):
def capture_archive_listing(
repository_path,
archive,
storage_config,
config,
local_borg_version,
global_arguments,
list_path=None,
@ -97,18 +97,18 @@ def capture_archive_listing(
remote_path=None,
):
'''
Given a local or remote repository path, an archive name, a storage config dict, the local Borg
Given a local or remote repository path, an archive name, a configuration dict, the local Borg
version, global arguments as an argparse.Namespace, the archive path in which to list files, and
local and remote Borg paths, capture the output of listing that archive and return it as a list
of file paths.
'''
borg_environment = environment.make_environment(storage_config)
borg_environment = environment.make_environment(config)
return tuple(
execute_command_and_capture_output(
make_list_command(
repository_path,
storage_config,
config,
local_borg_version,
argparse.Namespace(
repository=repository_path,
@ -131,7 +131,7 @@ def capture_archive_listing(
def list_archive(
repository_path,
storage_config,
config,
local_borg_version,
list_arguments,
global_arguments,
@ -139,7 +139,7 @@ def list_archive(
remote_path=None,
):
'''
Given a local or remote repository path, a storage config dict, the local Borg version, global
Given a local or remote repository path, a configuration dict, the local Borg version, global
arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace,
and local and remote Borg paths, display the output of listing the files of a Borg archive (or
return JSON output). If list_arguments.find_paths are given, list the files by searching across
@ -167,7 +167,7 @@ def list_archive(
)
return rlist.list_repository(
repository_path,
storage_config,
config,
local_borg_version,
rlist_arguments,
global_arguments,
@ -187,7 +187,7 @@ def list_archive(
'The --json flag on the list action is not supported when using the --archive/--find flags.'
)
borg_environment = environment.make_environment(storage_config)
borg_environment = environment.make_environment(config)
# If there are any paths to find (and there's not a single archive already selected), start by
# getting a list of archives to search.
@ -209,7 +209,7 @@ def list_archive(
execute_command_and_capture_output(
rlist.make_rlist_command(
repository_path,
storage_config,
config,
local_borg_version,
rlist_arguments,
global_arguments,
@ -238,7 +238,7 @@ def list_archive(
main_command = make_list_command(
repository_path,
storage_config,
config,
local_borg_version,
archive_arguments,
global_arguments,

8
borgmatic/borg/mount.py
View File

@ -10,7 +10,7 @@ def mount_archive(
repository_path,
archive,
mount_arguments,
storage_config,
config,
local_borg_version,
global_arguments,
local_path='borg',
@ -22,8 +22,8 @@ def mount_archive(
dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional
local and remote Borg paths, mount the archive onto the mount point.
'''
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None)
full_command = (
(local_path, 'mount')
@ -58,7 +58,7 @@ def mount_archive(
+ (tuple(mount_arguments.paths) if mount_arguments.paths else ())
)
borg_environment = environment.make_environment(storage_config)
borg_environment = environment.make_environment(config)
# Don't capture the output when foreground mode is used so that ctrl-C can work properly.
if mount_arguments.foreground:

34
borgmatic/borg/prune.py
View File

@ -7,9 +7,9 @@ from borgmatic.execute import execute_command
logger = logging.getLogger(__name__)
def make_prune_flags(storage_config, retention_config, local_borg_version):
def make_prune_flags(config, local_borg_version):
'''
Given a retention config dict mapping from option name to value, transform it into an sequence of
Given a configuration dict mapping from option name to value, transform it into an sequence of
command-line flags.
For example, given a retention config of:
@ -23,12 +23,12 @@ def make_prune_flags(storage_config, retention_config, local_borg_version):
('--keep-monthly', '6'),
)
'''
config = retention_config.copy()
prefix = config.pop('prefix', None)
flag_pairs = (
('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
('--' + option_name.replace('_', '-'), str(value))
for option_name, value in config.items()
if option_name.startswith('keep_')
)
prefix = config.get('prefix')
return tuple(element for pair in flag_pairs for element in pair) + (
(
@ -39,8 +39,8 @@ def make_prune_flags(storage_config, retention_config, local_borg_version):
if prefix
else (
flags.make_match_archives_flags(
storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
config.get('match_archives'),
config.get('archive_name_format'),
local_borg_version,
)
)
@ -50,8 +50,7 @@ def make_prune_flags(storage_config, retention_config, local_borg_version):
def prune_archives(
dry_run,
repository_path,
storage_config,
retention_config,
config,
local_borg_version,
prune_arguments,
global_arguments,
@ -59,18 +58,17 @@ def prune_archives(
remote_path=None,
):
'''
Given dry-run flag, a local or remote repository path, a storage config dict, and a
retention config dict, prune Borg archives according to the retention policy specified in that
configuration.
Given dry-run flag, a local or remote repository path, and a configuration dict, prune Borg
archives according to the retention policy specified in that configuration.
'''
borgmatic.logger.add_custom_log_levels()
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
extra_borg_options = storage_config.get('extra_borg_options', {}).get('prune', '')
umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None)
extra_borg_options = config.get('extra_borg_options', {}).get('prune', '')
full_command = (
(local_path, 'prune')
+ make_prune_flags(storage_config, retention_config, local_borg_version)
+ make_prune_flags(config, local_borg_version)
+ (('--remote-path', remote_path) if remote_path else ())
+ (('--umask', str(umask)) if umask else ())
+ (('--log-json',) if global_arguments.log_json else ())
@ -97,5 +95,5 @@ def prune_archives(
full_command,
output_log_level=output_log_level,
borg_local_path=local_path,
extra_environment=environment.make_environment(storage_config),
extra_environment=environment.make_environment(config),
)

16
borgmatic/borg/rcreate.py
View File

@ -14,7 +14,7 @@ RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
def create_repository(
dry_run,
repository_path,
storage_config,
config,
local_borg_version,
global_arguments,
encryption_mode,
@ -27,15 +27,15 @@ def create_repository(
remote_path=None,
):
'''
Given a dry-run flag, a local or remote repository path, a storage configuration dict, the local
Borg version, a Borg encryption mode, the path to another repo whose key material should be
reused, whether the repository should be append-only, and the storage quota to use, create the
Given a dry-run flag, a local or remote repository path, a configuration dict, the local Borg
version, a Borg encryption mode, the path to another repo whose key material should be reused,
whether the repository should be append-only, and the storage quota to use, create the
repository. If the repository already exists, then log and skip creation.
'''
try:
rinfo.display_repository_info(
repository_path,
storage_config,
config,
local_borg_version,
argparse.Namespace(json=True),
global_arguments,
@ -48,8 +48,8 @@ def create_repository(
if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
raise
lock_wait = storage_config.get('lock_wait')
extra_borg_options = storage_config.get('extra_borg_options', {}).get('rcreate', '')
lock_wait = config.get('lock_wait')
extra_borg_options = config.get('extra_borg_options', {}).get('rcreate', '')
rcreate_command = (
(local_path,)
@ -82,5 +82,5 @@ def create_repository(
rcreate_command,
output_file=DO_NOT_CAPTURE,
borg_local_path=local_path,
extra_environment=environment.make_environment(storage_config),
extra_environment=environment.make_environment(config),
)

8
borgmatic/borg/rinfo.py
View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
def display_repository_info(
repository_path,
storage_config,
config,
local_borg_version,
rinfo_arguments,
global_arguments,
@ -17,12 +17,12 @@ def display_repository_info(
remote_path=None,
):
'''
Given a local or remote repository path, a storage config dict, the local Borg version, the
Given a local or remote repository path, a configuration dict, the local Borg version, the
arguments to the rinfo action, and global arguments as an argparse.Namespace, display summary
information for the Borg repository or return JSON summary information.
'''
borgmatic.logger.add_custom_log_levels()
lock_wait = storage_config.get('lock_wait', None)
lock_wait = config.get('lock_wait', None)
full_command = (
(local_path,)
@ -48,7 +48,7 @@ def display_repository_info(
+ flags.make_repository_flags(repository_path, local_borg_version)
)
extra_environment = environment.make_environment(storage_config)
extra_environment = environment.make_environment(config)
if rinfo_arguments.json:
return execute_command_and_capture_output(

26
borgmatic/borg/rlist.py
View File

@ -10,14 +10,14 @@ logger = logging.getLogger(__name__)
def resolve_archive_name(
repository_path,
archive,
storage_config,
config,
local_borg_version,
global_arguments,
local_path='borg',
remote_path=None,
):
'''
Given a local or remote repository path, an archive name, a storage config dict, the local Borg
Given a local or remote repository path, an archive name, a configuration dict, the local Borg
version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path,
return the archive name. But if the archive name is "latest", then instead introspect the
repository for the latest archive and return its name.
@ -34,7 +34,7 @@ def resolve_archive_name(
)
+ flags.make_flags('remote-path', remote_path)
+ flags.make_flags('log-json', global_arguments.log_json)
+ flags.make_flags('lock-wait', storage_config.get('lock_wait'))
+ flags.make_flags('lock-wait', config.get('lock_wait'))
+ flags.make_flags('last', 1)
+ ('--short',)
+ flags.make_repository_flags(repository_path, local_borg_version)
@ -42,7 +42,7 @@ def resolve_archive_name(
output = execute_command_and_capture_output(
full_command,
extra_environment=environment.make_environment(storage_config),
extra_environment=environment.make_environment(config),
)
try:
latest_archive = output.strip().splitlines()[-1]
@ -59,7 +59,7 @@ MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
def make_rlist_command(
repository_path,
storage_config,
config,
local_borg_version,
rlist_arguments,
global_arguments,
@ -67,7 +67,7 @@ def make_rlist_command(
remote_path=None,
):
'''
Given a local or remote repository path, a storage config dict, the local Borg version, the
Given a local or remote repository path, a configuration dict, the local Borg version, the
arguments to the rlist action, global arguments as an argparse.Namespace instance, and local and
remote Borg paths, return a command as a tuple to list archives with a repository.
'''
@ -88,7 +88,7 @@ def make_rlist_command(
)
+ flags.make_flags('remote-path', remote_path)
+ flags.make_flags('log-json', global_arguments.log_json)
+ flags.make_flags('lock-wait', storage_config.get('lock_wait'))
+ flags.make_flags('lock-wait', config.get('lock_wait'))
+ (
(
flags.make_flags('match-archives', f'sh:{rlist_arguments.prefix}*')
@ -98,8 +98,8 @@ def make_rlist_command(
if rlist_arguments.prefix
else (
flags.make_match_archives_flags(
rlist_arguments.match_archives or storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
rlist_arguments.match_archives or config.get('match_archives'),
config.get('archive_name_format'),
local_borg_version,
)
)
@ -111,7 +111,7 @@ def make_rlist_command(
def list_repository(
repository_path,
storage_config,
config,
local_borg_version,
rlist_arguments,
global_arguments,
@ -119,17 +119,17 @@ def list_repository(
remote_path=None,
):
'''
Given a local or remote repository path, a storage config dict, the local Borg version, the
Given a local or remote repository path, a configuration dict, the local Borg version, the
arguments to the list action, global arguments as an argparse.Namespace instance, and local and
remote Borg paths, display the output of listing Borg archives in the given repository (or
return JSON output).
'''
borgmatic.logger.add_custom_log_levels()
borg_environment = environment.make_environment(storage_config)
borg_environment = environment.make_environment(config)
main_command = make_rlist_command(
repository_path,
storage_config,
config,
local_borg_version,
rlist_arguments,
global_arguments,

12
borgmatic/borg/transfer.py
View File

@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
def transfer_archives(
dry_run,
repository_path,
storage_config,
config,
local_borg_version,
transfer_arguments,
global_arguments,
@ -18,7 +18,7 @@ def transfer_archives(
remote_path=None,
):
'''
Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg
Given a dry-run flag, a local or remote repository path, a configuration dict, the local Borg
version, the arguments to the transfer action, and global arguments as an argparse.Namespace
instance, transfer archives to the given repository.
'''
@ -30,7 +30,7 @@ def transfer_archives(
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ flags.make_flags('remote-path', remote_path)
+ flags.make_flags('log-json', global_arguments.log_json)
+ flags.make_flags('lock-wait', storage_config.get('lock_wait', None))
+ flags.make_flags('lock-wait', config.get('lock_wait', None))
+ (
flags.make_flags_from_arguments(
transfer_arguments,
@ -40,8 +40,8 @@ def transfer_archives(
flags.make_match_archives_flags(
transfer_arguments.match_archives
or transfer_arguments.archive
or storage_config.get('match_archives'),
storage_config.get('archive_name_format'),
or config.get('match_archives'),
config.get('archive_name_format'),
local_borg_version,
)
)
@ -56,5 +56,5 @@ def transfer_archives(
output_log_level=logging.ANSWER,
output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
borg_local_path=local_path,
extra_environment=environment.make_environment(storage_config),
extra_environment=environment.make_environment(config),
)

6
borgmatic/borg/version.py
View File

@ -6,9 +6,9 @@ from borgmatic.execute import execute_command_and_capture_output
logger = logging.getLogger(__name__)
def local_borg_version(storage_config, local_path='borg'):
def local_borg_version(config, local_path='borg'):
'''
Given a storage configuration dict and a local Borg binary path, return a version string for it.
Given a configuration dict and a local Borg binary path, return a version string for it.
Raise OSError or CalledProcessError if there is a problem running Borg.
Raise ValueError if the version cannot be parsed.
@ -20,7 +20,7 @@ def local_borg_version(storage_config, local_path='borg'):
)
output = execute_command_and_capture_output(
full_command,
extra_environment=environment.make_environment(storage_config),
extra_environment=environment.make_environment(config),
)
try:

2
borgmatic/commands/arguments.py
View File

@ -330,7 +330,7 @@ def make_parsers():
)
global_group.add_argument(
'--override',
metavar='SECTION.OPTION=VALUE',
metavar='OPTION.SUBOPTION=VALUE',
nargs='+',
dest='overrides',
action='extend',

117
borgmatic/commands/borgmatic.py
View File

@ -58,16 +58,12 @@ def run_configuration(config_filename, config, arguments):
* JSON output strings from successfully executing any actions that produce JSON
* logging.LogRecord instances containing errors from any actions or backup hooks that fail
'''
(location, storage, retention, consistency, hooks) = (
config.get(section_name, {})
for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
)
global_arguments = arguments['global']
local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path')
retries = storage.get('retries', 0)
retry_wait = storage.get('retry_wait', 0)
local_path = config.get('local_path', 'borg')
remote_path = config.get('remote_path')
retries = config.get('retries', 0)
retry_wait = config.get('retry_wait', 0)
encountered_error = None
error_repository = ''
using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments)
@ -75,7 +71,7 @@ def run_configuration(config_filename, config, arguments):
monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED
try:
local_borg_version = borg_version.local_borg_version(storage, local_path)
local_borg_version = borg_version.local_borg_version(config, local_path)
except (OSError, CalledProcessError, ValueError) as error:
yield from log_error_records(f'{config_filename}: Error getting local Borg version', error)
return
@ -84,7 +80,7 @@ def run_configuration(config_filename, config, arguments):
if monitoring_hooks_are_activated:
dispatch.call_hooks(
'initialize_monitor',
hooks,
config,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitoring_log_level,
@ -93,7 +89,7 @@ def run_configuration(config_filename, config, arguments):
dispatch.call_hooks(
'ping_monitor',
hooks,
config,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitor.State.START,
@ -109,7 +105,7 @@ def run_configuration(config_filename, config, arguments):
if not encountered_error:
repo_queue = Queue()
for repo in location['repositories']:
for repo in config['repositories']:
repo_queue.put(
(repo, 0),
)
@ -129,11 +125,7 @@ def run_configuration(config_filename, config, arguments):
yield from run_actions(
arguments=arguments,
config_filename=config_filename,
location=location,
storage=storage,
retention=retention,
consistency=consistency,
hooks=hooks,
config=config,
local_path=local_path,
remote_path=remote_path,
local_borg_version=local_borg_version,
@ -172,7 +164,7 @@ def run_configuration(config_filename, config, arguments):
# send logs irrespective of error
dispatch.call_hooks(
'ping_monitor',
hooks,
config,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitor.State.LOG,
@ -191,7 +183,7 @@ def run_configuration(config_filename, config, arguments):
if monitoring_hooks_are_activated:
dispatch.call_hooks(
'ping_monitor',
hooks,
config,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitor.State.FINISH,
@ -200,7 +192,7 @@ def run_configuration(config_filename, config, arguments):
)
dispatch.call_hooks(
'destroy_monitor',
hooks,
config,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitoring_log_level,
@ -216,8 +208,8 @@ def run_configuration(config_filename, config, arguments):
if encountered_error and using_primary_action:
try:
command.execute_hook(
hooks.get('on_error'),
hooks.get('umask'),
config.get('on_error'),
config.get('umask'),
config_filename,
'on-error',
global_arguments.dry_run,
@ -227,7 +219,7 @@ def run_configuration(config_filename, config, arguments):
)
dispatch.call_hooks(
'ping_monitor',
hooks,
config,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitor.State.FAIL,
@ -236,7 +228,7 @@ def run_configuration(config_filename, config, arguments):
)
dispatch.call_hooks(
'destroy_monitor',
hooks,
config,
config_filename,
monitor.MONITOR_HOOK_NAMES,
monitoring_log_level,
@ -253,11 +245,7 @@ def run_actions(
*,
arguments,
config_filename,
location,
storage,
retention,
consistency,
hooks,
config,
local_path,
remote_path,
local_borg_version,
@ -282,13 +270,13 @@ def run_actions(
hook_context = {
'repository': repository_path,
# Deprecated: For backwards compatibility with borgmatic < 1.6.0.
'repositories': ','.join([repo['path'] for repo in location['repositories']]),
'repositories': ','.join([repo['path'] for repo in config['repositories']]),
'log_file': global_arguments.log_file if global_arguments.log_file else '',
}
command.execute_hook(
hooks.get('before_actions'),
hooks.get('umask'),
config.get('before_actions'),
config.get('umask'),
config_filename,
'pre-actions',
global_arguments.dry_run,
@ -299,7 +287,7 @@ def run_actions(
if action_name == 'rcreate':
borgmatic.actions.rcreate.run_rcreate(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -309,7 +297,7 @@ def run_actions(
elif action_name == 'transfer':
borgmatic.actions.transfer.run_transfer(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -320,9 +308,7 @@ def run_actions(
yield from borgmatic.actions.create.run_create(
config_filename,
repository,
location,
storage,
hooks,
config,
hook_context,
local_borg_version,
action_arguments,
@ -335,9 +321,7 @@ def run_actions(
borgmatic.actions.prune.run_prune(
config_filename,
repository,
storage,
retention,
hooks,
config,
hook_context,
local_borg_version,
action_arguments,
@ -350,9 +334,7 @@ def run_actions(
borgmatic.actions.compact.run_compact(
config_filename,
repository,
storage,
retention,
hooks,
config,
hook_context,
local_borg_version,
action_arguments,
@ -362,14 +344,11 @@ def run_actions(
remote_path,
)
elif action_name == 'check':
if checks.repository_enabled_for_checks(repository, consistency):
if checks.repository_enabled_for_checks(repository, config):
borgmatic.actions.check.run_check(
config_filename,
repository,
location,
storage,
consistency,
hooks,
config,
hook_context,
local_borg_version,
action_arguments,
@ -381,9 +360,7 @@ def run_actions(
borgmatic.actions.extract.run_extract(
config_filename,
repository,
location,
storage,
hooks,
config,
hook_context,
local_borg_version,
action_arguments,
@ -394,7 +371,7 @@ def run_actions(
elif action_name == 'export-tar':
borgmatic.actions.export_tar.run_export_tar(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -404,7 +381,7 @@ def run_actions(
elif action_name == 'mount':
borgmatic.actions.mount.run_mount(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -414,9 +391,7 @@ def run_actions(
elif action_name == 'restore':
borgmatic.actions.restore.run_restore(
repository,
location,
storage,
hooks,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -426,7 +401,7 @@ def run_actions(
elif action_name == 'rlist':
yield from borgmatic.actions.rlist.run_rlist(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -436,7 +411,7 @@ def run_actions(
elif action_name == 'list':
yield from borgmatic.actions.list.run_list(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -446,7 +421,7 @@ def run_actions(
elif action_name == 'rinfo':
yield from borgmatic.actions.rinfo.run_rinfo(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -456,7 +431,7 @@ def run_actions(
elif action_name == 'info':
yield from borgmatic.actions.info.run_info(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -466,7 +441,7 @@ def run_actions(
elif action_name == 'break-lock':
borgmatic.actions.break_lock.run_break_lock(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -476,7 +451,7 @@ def run_actions(
elif action_name == 'borg':
borgmatic.actions.borg.run_borg(
repository,
storage,
config,
local_borg_version,
action_arguments,
global_arguments,
@ -485,8 +460,8 @@ def run_actions(
)
command.execute_hook(
hooks.get('after_actions'),
hooks.get('umask'),
config.get('after_actions'),
config.get('umask'),
config_filename,
'post-actions',
global_arguments.dry_run,
@ -613,7 +588,7 @@ def get_local_path(configs):
Arbitrarily return the local path from the first configuration dict. Default to "borg" if not
set.
'''
return next(iter(configs.values())).get('location', {}).get('local_path', 'borg')
return next(iter(configs.values())).get('local_path', 'borg')
def collect_highlander_action_summary_logs(configs, arguments, configuration_parse_errors):
@ -627,6 +602,8 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
A highlander action is an action that cannot coexist with other actions on the borgmatic
command-line, and borgmatic exits after processing such an action.
'''
add_custom_log_levels()
if 'bootstrap' in arguments:
try:
# No configuration file is needed for bootstrap.
@ -744,10 +721,9 @@ def collect_configuration_run_summary_logs(configs, arguments):
if 'create' in arguments:
try:
for config_filename, config in configs.items():
hooks = config.get('hooks', {})
command.execute_hook(
hooks.get('before_everything'),
hooks.get('umask'),
config.get('before_everything'),
config.get('umask'),
config_filename,
'pre-everything',
arguments['global'].dry_run,
@ -792,10 +768,9 @@ def collect_configuration_run_summary_logs(configs, arguments):
if 'create' in arguments:
try:
for config_filename, config in configs.items():
hooks = config.get('hooks', {})
command.execute_hook(
hooks.get('after_everything'),
hooks.get('umask'),
config.get('after_everything'),
config.get('umask'),
config_filename,
'post-everything',
arguments['global'].dry_run,

27
borgmatic/config/generate.py
View File

@ -11,7 +11,7 @@ INDENT = 4
SEQUENCE_INDENT = 2
def _insert_newline_before_comment(config, field_name):
def insert_newline_before_comment(config, field_name):
'''
Using some ruamel.yaml black magic, insert a blank line in the config right before the given
field and its comments.
@ -21,10 +21,10 @@ def _insert_newline_before_comment(config, field_name):
)
def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
'''
Given a loaded configuration schema, generate and return sample config for it. Include comments
for each section based on the schema "description".
for each option based on the schema "description".
'''
schema_type = schema.get('type')
example = schema.get('example')
@ -33,13 +33,13 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
if schema_type == 'array':
config = yaml.comments.CommentedSeq(
[_schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
[schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
)
add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
elif schema_type == 'object':
config = yaml.comments.CommentedMap(
[
(field_name, _schema_to_sample_configuration(sub_schema, level + 1))
(field_name, schema_to_sample_configuration(sub_schema, level + 1))
for field_name, sub_schema in schema['properties'].items()
]
)
@ -53,13 +53,13 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
return config
def _comment_out_line(line):
def comment_out_line(line):
# If it's already is commented out (or empty), there's nothing further to do!
stripped_line = line.lstrip()
if not stripped_line or stripped_line.startswith('#'):
return line
# Comment out the names of optional sections, inserting the '#' after any indent for aesthetics.
# Comment out the names of optional options, inserting the '#' after any indent for aesthetics.
matches = re.match(r'(\s*)', line)
indent_spaces = matches.group(0) if matches else ''
count_indent_spaces = len(indent_spaces)
@ -67,7 +67,7 @@ def _comment_out_line(line):
return '# '.join((indent_spaces, line[count_indent_spaces:]))
def _comment_out_optional_configuration(rendered_config):
def comment_out_optional_configuration(rendered_config):
'''
Post-process a rendered configuration string to comment out optional key/values, as determined
by a sentinel in the comment before each key.
@ -92,7 +92,7 @@ def _comment_out_optional_configuration(rendered_config):
if not line.strip():
optional = False
lines.append(_comment_out_line(line) if optional else line)
lines.append(comment_out_line(line) if optional else line)
return '\n'.join(lines)
@ -165,7 +165,6 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
return
REQUIRED_SECTION_NAMES = {'location', 'retention'}
REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
@ -185,7 +184,7 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa
# If this is an optional key, add an indicator to the comment flagging it to be commented
# out from the sample configuration. This sentinel is consumed by downstream processing that
# does the actual commenting out.
if field_name not in REQUIRED_SECTION_NAMES and field_name not in REQUIRED_KEYS:
if field_name not in REQUIRED_KEYS:
description = (
'\n'.join((description, COMMENTED_OUT_SENTINEL))
if description
@ -199,7 +198,7 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa
config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
if index > 0:
_insert_newline_before_comment(config, field_name)
insert_newline_before_comment(config, field_name)
RUAMEL_YAML_COMMENTS_INDEX = 1
@ -284,7 +283,7 @@ def generate_sample_configuration(
normalize.normalize(source_filename, source_config)
destination_config = merge_source_configuration_into_destination(
_schema_to_sample_configuration(schema), source_config
schema_to_sample_configuration(schema), source_config
)
if dry_run:
@ -292,6 +291,6 @@ def generate_sample_configuration(
write_configuration(
destination_filename,
_comment_out_optional_configuration(render_configuration(destination_config)),
comment_out_optional_configuration(render_configuration(destination_config)),
overwrite=overwrite,
)

10
borgmatic/config/load.py
View File

@ -97,8 +97,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
```
These includes are deep merged into the current configuration file. For instance, in this
example, any "retention" options in common.yaml will get merged into the "retention" section
in the example configuration file.
example, any "option" with sub-options in common.yaml will get merged into the corresponding
"option" with sub-options in the example configuration file.
'''
representer = ruamel.yaml.representer.SafeRepresenter()
@ -116,7 +116,7 @@ def load_configuration(filename):
'''
Load the given configuration file and return its contents as a data structure of nested dicts
and lists. Also, replace any "{constant}" strings with the value of the "constant" key in the
"constants" section of the configuration file.
"constants" option of the configuration file.
Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
if there are too many recursive includes.
@ -223,8 +223,8 @@ def deep_merge_nodes(nodes):
If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged.
The purpose of deep merging like this is to support, for instance, merging one borgmatic
configuration file into another for reuse, such that a configuration section ("retention",
etc.) does not completely replace the corresponding section in a merged file.
configuration file into another for reuse, such that a configuration option with sub-options
does not completely replace the corresponding option in a merged file.
Raise ValueError if a merge is implied using two incompatible types.
'''

119
borgmatic/config/normalize.py
View File

@ -2,21 +2,70 @@ import logging
import os
def normalize_sections(config_filename, config):
'''
Given a configuration filename and a configuration dict of its loaded contents, airlift any
options out of sections ("location:", etc.) to the global scope and delete those sections.
Return any log message warnings produced based on the normalization performed.
Raise ValueError if the "prefix" option is set in both "location" and "consistency" sections.
'''
location = config.get('location') or {}
storage = config.get('storage') or {}
consistency = config.get('consistency') or {}
hooks = config.get('hooks') or {}
if (
location.get('prefix')
and consistency.get('prefix')
and location.get('prefix') != consistency.get('prefix')
):
raise ValueError(
'The retention prefix and the consistency prefix cannot have different values (unless one is not set).'
)
if storage.get('umask') and hooks.get('umask') and storage.get('umask') != hooks.get('umask'):
raise ValueError(
'The storage umask and the hooks umask cannot have different values (unless one is not set).'
)
any_section_upgraded = False
# Move any options from deprecated sections into the global scope.
for section_name in ('location', 'storage', 'retention', 'consistency', 'output', 'hooks'):
section_config = config.get(section_name)
if section_config:
any_section_upgraded = True
del config[section_name]
config.update(section_config)
if any_section_upgraded:
return [
logging.makeLogRecord(
dict(
levelno=logging.WARNING,
levelname='WARNING',
msg=f'{config_filename}: Configuration sections like location: and storage: are deprecated and support will be removed from a future release. Move all of your options out of sections to the global scope.',
)
)
]
return []
def normalize(config_filename, config):
'''
Given a configuration filename and a configuration dict of its loaded contents, apply particular
hard-coded rules to normalize the configuration to adhere to the current schema. Return any log
message warnings produced based on the normalization performed.
Raise ValueError the configuration cannot be normalized.
'''
logs = []
location = config.get('location') or {}
storage = config.get('storage') or {}
consistency = config.get('consistency') or {}
retention = config.get('retention') or {}
hooks = config.get('hooks') or {}
logs = normalize_sections(config_filename, config)
# Upgrade exclude_if_present from a string to a list.
exclude_if_present = location.get('exclude_if_present')
exclude_if_present = config.get('exclude_if_present')
if isinstance(exclude_if_present, str):
logs.append(
logging.makeLogRecord(
@ -27,10 +76,10 @@ def normalize(config_filename, config):
)
)
)
config['location']['exclude_if_present'] = [exclude_if_present]
config['exclude_if_present'] = [exclude_if_present]
# Upgrade various monitoring hooks from a string to a dict.
healthchecks = hooks.get('healthchecks')
healthchecks = config.get('healthchecks')
if isinstance(healthchecks, str):
logs.append(
logging.makeLogRecord(
@ -41,9 +90,9 @@ def normalize(config_filename, config):
)
)
)
config['hooks']['healthchecks'] = {'ping_url': healthchecks}
config['healthchecks'] = {'ping_url': healthchecks}
cronitor = hooks.get('cronitor')
cronitor = config.get('cronitor')
if isinstance(cronitor, str):
logs.append(
logging.makeLogRecord(
@ -54,9 +103,9 @@ def normalize(config_filename, config):
)
)
)
config['hooks']['cronitor'] = {'ping_url': cronitor}
config['cronitor'] = {'ping_url': cronitor}
pagerduty = hooks.get('pagerduty')
pagerduty = config.get('pagerduty')
if isinstance(pagerduty, str):
logs.append(
logging.makeLogRecord(
@ -67,9 +116,9 @@ def normalize(config_filename, config):
)
)
)
config['hooks']['pagerduty'] = {'integration_key': pagerduty}
config['pagerduty'] = {'integration_key': pagerduty}
cronhub = hooks.get('cronhub')
cronhub = config.get('cronhub')
if isinstance(cronhub, str):
logs.append(
logging.makeLogRecord(
@ -80,10 +129,10 @@ def normalize(config_filename, config):
)
)
)
config['hooks']['cronhub'] = {'ping_url': cronhub}
config['cronhub'] = {'ping_url': cronhub}
# Upgrade consistency checks from a list of strings to a list of dicts.
checks = consistency.get('checks')
checks = config.get('checks')
if isinstance(checks, list) and len(checks) and isinstance(checks[0], str):
logs.append(
logging.makeLogRecord(
@ -94,10 +143,10 @@ def normalize(config_filename, config):
)
)
)
config['consistency']['checks'] = [{'name': check_type} for check_type in checks]
config['checks'] = [{'name': check_type} for check_type in checks]
# Rename various configuration options.
numeric_owner = location.pop('numeric_owner', None)
numeric_owner = config.pop('numeric_owner', None)
if numeric_owner is not None:
logs.append(
logging.makeLogRecord(
@ -108,9 +157,9 @@ def normalize(config_filename, config):
)
)
)
config['location']['numeric_ids'] = numeric_owner
config['numeric_ids'] = numeric_owner
bsd_flags = location.pop('bsd_flags', None)
bsd_flags = config.pop('bsd_flags', None)
if bsd_flags is not None:
logs.append(
logging.makeLogRecord(
@ -121,9 +170,9 @@ def normalize(config_filename, config):
)
)
)
config['location']['flags'] = bsd_flags
config['flags'] = bsd_flags
remote_rate_limit = storage.pop('remote_rate_limit', None)
remote_rate_limit = config.pop('remote_rate_limit', None)
if remote_rate_limit is not None:
logs.append(
logging.makeLogRecord(
@ -134,10 +183,10 @@ def normalize(config_filename, config):
)
)
)
config['storage']['upload_rate_limit'] = remote_rate_limit
config['upload_rate_limit'] = remote_rate_limit
# Upgrade remote repositories to ssh:// syntax, required in Borg 2.
repositories = location.get('repositories')
repositories = config.get('repositories')
if repositories:
if isinstance(repositories[0], str):
logs.append(
@ -149,11 +198,11 @@ def normalize(config_filename, config):
)
)
)
config['location']['repositories'] = [
{'path': repository} for repository in repositories
]
repositories = config['location']['repositories']
config['location']['repositories'] = []
config['repositories'] = [{'path': repository} for repository in repositories]
repositories = config['repositories']
config['repositories'] = []
for repository_dict in repositories:
repository_path = repository_dict['path']
if '~' in repository_path:
@ -171,14 +220,14 @@ def normalize(config_filename, config):
updated_repository_path = os.path.abspath(
repository_path.partition('file://')[-1]
)
config['location']['repositories'].append(
config['repositories'].append(
dict(
repository_dict,
path=updated_repository_path,
)
)
elif repository_path.startswith('ssh://'):
config['location']['repositories'].append(repository_dict)
config['repositories'].append(repository_dict)
else:
rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}"
logs.append(
@ -190,16 +239,16 @@ def normalize(config_filename, config):
)
)
)
config['location']['repositories'].append(
config['repositories'].append(
dict(
repository_dict,
path=rewritten_repository_path,
)
)
else:
config['location']['repositories'].append(repository_dict)
config['repositories'].append(repository_dict)
if consistency.get('prefix') or retention.get('prefix'):
if config.get('prefix'):
logs.append(
logging.makeLogRecord(
dict(

28
borgmatic/config/override.py
View File

@ -32,19 +32,33 @@ def convert_value_type(value):
return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
LEGACY_SECTION_NAMES = {'location', 'storage', 'retention', 'consistency', 'output', 'hooks'}
def strip_section_names(parsed_override_key):
'''
Given a parsed override key as a tuple of option and suboption names, strip out any initial
legacy section names, since configuration file normalization also strips them out.
'''
if parsed_override_key[0] in LEGACY_SECTION_NAMES:
return parsed_override_key[1:]
return parsed_override_key
def parse_overrides(raw_overrides):
'''
Given a sequence of configuration file override strings in the form of "section.option=value",
Given a sequence of configuration file override strings in the form of "option.suboption=value",
parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For
instance, given the following raw overrides:
['section.my_option=value1', 'section.other_option=value2']
['my_option.suboption=value1', 'other_option=value2']
... return this:
(
(('section', 'my_option'), 'value1'),
(('section', 'other_option'), 'value2'),
(('my_option', 'suboption'), 'value1'),
(('other_option'), 'value2'),
)
Raise ValueError if an override can't be parsed.
@ -59,13 +73,13 @@ def parse_overrides(raw_overrides):
raw_keys, value = raw_override.split('=', 1)
parsed_overrides.append(
(
tuple(raw_keys.split('.')),
strip_section_names(tuple(raw_keys.split('.'))),
convert_value_type(value),
)
)
except ValueError:
raise ValueError(
f"Invalid override '{raw_override}'. Make sure you use the form: SECTION.OPTION=VALUE"
f"Invalid override '{raw_override}'. Make sure you use the form: OPTION=VALUE or OPTION.SUBOPTION=VALUE"
)
except ruamel.yaml.error.YAMLError as error:
raise ValueError(f"Invalid override '{raw_override}': {error.problem}")
@ -76,7 +90,7 @@ def parse_overrides(raw_overrides):
def apply_overrides(config, raw_overrides):
'''
Given a configuration dict and a sequence of configuration file override strings in the form of
"section.option=value", parse each override and set it the configuration dict.
"option.suboption=value", parse each override and set it the configuration dict.
'''
overrides = parse_overrides(raw_overrides)

2577
borgmatic/config/schema.yaml
View File

File diff suppressed because it is too large Load Diff

25
borgmatic/config/validate.py
View File

@ -71,18 +71,15 @@ def apply_logical_validation(config_filename, parsed_configuration):
below), run through any additional logical validation checks. If there are any such validation
problems, raise a Validation_error.
'''
location_repositories = parsed_configuration.get('location', {}).get('repositories')
check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', [])
repositories = parsed_configuration.get('repositories')
check_repositories = parsed_configuration.get('check_repositories', [])
for repository in check_repositories:
if not any(
repositories_match(repository, config_repository)
for config_repository in location_repositories
repositories_match(repository, config_repository) for config_repository in repositories
):
raise Validation_error(
config_filename,
(
f'Unknown repository in the "consistency" section\'s "check_repositories": {repository}',
),
(f'Unknown repository in "check_repositories": {repository}',),
)
@ -90,11 +87,15 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
'''
Given the path to a config filename in YAML format, the path to a schema filename in a YAML
rendition of JSON Schema format, a sequence of configuration file override strings in the form
of "section.option=value", return the parsed configuration as a data structure of nested dicts
of "option.suboption=value", return the parsed configuration as a data structure of nested dicts
and lists corresponding to the schema. Example return value:
{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
{
'source_directories': ['/home', '/etc'],
'repository': 'hostname.borg',
'keep_daily': 7,
'checks': ['repository', 'archives'],
}
Also return a sequence of logging.LogRecord instances containing any warnings about the
configuration.
@ -174,7 +175,7 @@ def guard_configuration_contains_repository(repository, configurations):
tuple(
config_repository
for config in configurations.values()
for config_repository in config['location']['repositories']
for config_repository in config['repositories']
if repositories_match(config_repository, repository)
)
)
@ -198,7 +199,7 @@ def guard_single_repository_selected(repository, configurations):
tuple(
config_repository
for config in configurations.values()
for config_repository in config['location']['repositories']
for config_repository in config['repositories']
)
)

2
borgmatic/hooks/cronhub.py
View File

@ -22,7 +22,7 @@ def initialize_monitor(
pass
def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
'''
Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration
filename in any log entries. If this is a dry run, then don't actually ping anything.

2
borgmatic/hooks/cronitor.py
View File

@ -22,7 +22,7 @@ def initialize_monitor(
pass
def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
'''
Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration
filename in any log entries. If this is a dry run, then don't actually ping anything.

39
borgmatic/hooks/dispatch.py
View File

@ -27,18 +27,17 @@ HOOK_NAME_TO_MODULE = {
}
def call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs):
def call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs):
'''
Given the hooks configuration dict and a prefix to use in log entries, call the requested
function of the Python module corresponding to the given hook name. Supply that call with the
configuration for this hook (if any), the log prefix, and any given args and kwargs. Return any
return value.
Given a configuration dict and a prefix to use in log entries, call the requested function of
the Python module corresponding to the given hook name. Supply that call with the configuration
for this hook (if any), the log prefix, and any given args and kwargs. Return any return value.
Raise ValueError if the hook name is unknown.
Raise AttributeError if the function name is not found in the module.
Raise anything else that the called function raises.
'''
config = hooks.get(hook_name, {})
hook_config = config.get(hook_name, {})
try:
module = HOOK_NAME_TO_MODULE[hook_name]
@ -46,15 +45,15 @@ def call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs):
raise ValueError(f'Unknown hook name: {hook_name}')
logger.debug(f'{log_prefix}: Calling {hook_name} hook function {function_name}')
return getattr(module, function_name)(config, log_prefix, *args, **kwargs)
return getattr(module, function_name)(hook_config, config, log_prefix, *args, **kwargs)
def call_hooks(function_name, hooks, log_prefix, hook_names, *args, **kwargs):
def call_hooks(function_name, config, log_prefix, hook_names, *args, **kwargs):
'''
Given the hooks configuration dict and a prefix to use in log entries, call the requested
function of the Python module corresponding to each given hook name. Supply each call with the
configuration for that hook, the log prefix, and any given args and kwargs. Collect any return
values into a dict from hook name to return value.
Given a configuration dict and a prefix to use in log entries, call the requested function of
the Python module corresponding to each given hook name. Supply each call with the configuration
for that hook, the log prefix, and any given args and kwargs. Collect any return values into a
dict from hook name to return value.
If the hook name is not present in the hooks configuration, then don't call the function for it
and omit it from the return values.
@ -64,23 +63,23 @@ def call_hooks(function_name, hooks, log_prefix, hook_names, *args, **kwargs):
Raise anything else that a called function raises. An error stops calls to subsequent functions.
'''
return {
hook_name: call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs)
hook_name: call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs)
for hook_name in hook_names
if hooks.get(hook_name)
if config.get(hook_name)
}
def call_hooks_even_if_unconfigured(function_name, hooks, log_prefix, hook_names, *args, **kwargs):
def call_hooks_even_if_unconfigured(function_name, config, log_prefix, hook_names, *args, **kwargs):
'''
Given the hooks configuration dict and a prefix to use in log entries, call the requested
function of the Python module corresponding to each given hook name. Supply each call with the
configuration for that hook, the log prefix, and any given args and kwargs. Collect any return
values into a dict from hook name to return value.
Given a configuration dict and a prefix to use in log entries, call the requested function of
the Python module corresponding to each given hook name. Supply each call with the configuration
for that hook, the log prefix, and any given args and kwargs. Collect any return values into a
dict from hook name to return value.
Raise AttributeError if the function name is not found in the module.
Raise anything else that a called function raises. An error stops calls to subsequent functions.
'''
return {
hook_name: call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs)
hook_name: call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs)
for hook_name in hook_names
}

2
borgmatic/hooks/healthchecks.py
View File

@ -90,7 +90,7 @@ def initialize_monitor(hook_config, config_filename, monitoring_log_level, dry_r
)
def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
'''
Ping the configured Healthchecks URL or UUID, modified with the monitor.State. Use the given
configuration filename in any log entries, and log to Healthchecks with the giving log level.

41
borgmatic/hooks/mongodb.py
View File

@ -6,21 +6,20 @@ from borgmatic.hooks import dump
logger = logging.getLogger(__name__)
def make_dump_path(location_config): # pragma: no cover
def make_dump_path(config): # pragma: no cover
'''
Make the dump path from the given location configuration and the name of this hook.
Make the dump path from the given configuration dict and the name of this hook.
'''
return dump.make_database_dump_path(
location_config.get('borgmatic_source_directory'), 'mongodb_databases'
config.get('borgmatic_source_directory'), 'mongodb_databases'
)
def dump_databases(databases, log_prefix, location_config, dry_run):
def dump_databases(databases, config, log_prefix, dry_run):
'''
Dump the given MongoDB databases to a named pipe. The databases are supplied as a sequence of
dicts, one dict describing each database as per the configuration schema. Use the given log
prefix in any log entries. Use the given location configuration dict to construct the
destination path.
dicts, one dict describing each database as per the configuration schema. Use the configuration
dict to construct the destination path and the given log prefix in any log entries.
Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
@ -33,7 +32,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
for database in databases:
name = database['name']
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), name, database.get('hostname')
make_dump_path(config), name, database.get('hostname')
)
dump_format = database.get('format', 'archive')
@ -82,35 +81,33 @@ def build_dump_command(database, dump_filename, dump_format):
return command
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
'''
Remove all database dump files for this hook regardless of the given databases. Use the log
prefix in any log entries. Use the given location configuration dict to construct the
destination path. If this is a dry run, then don't actually remove anything.
prefix in any log entries. Use the given configuration dict to construct the destination path.
If this is a dry run, then don't actually remove anything.
'''
dump.remove_database_dumps(make_dump_path(location_config), 'MongoDB', log_prefix, dry_run)
dump.remove_database_dumps(make_dump_path(config), 'MongoDB', log_prefix, dry_run)
def make_database_dump_pattern(
databases, log_prefix, location_config, name=None
): # pragma: no cover
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
'''
Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
Given a sequence of database configurations dicts, a configuration dict, a prefix to log with,
and a database name to match, return the corresponding glob patterns to match the database dump
in an archive.
'''
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*')
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
database_config, config, log_prefix, dry_run, extract_process, connection_params
):
'''
Restore the given MongoDB database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
output to consume.
Use the configuration dict to construct the destination path and the given log prefix in any log
entries. If this is a dry run, then don't actually restore anything. Trigger the given active
extract process (an instance of subprocess.Popen) to produce output to consume.
If the extract process is None, then restore the dump from the filesystem rather than from an
extract stream.
@ -122,7 +119,7 @@ def restore_database_dump(
database = database_config[0]
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'], database.get('hostname')
make_dump_path(config), database['name'], database.get('hostname')
)
restore_command = build_restore_command(
extract_process, database, dump_filename, connection_params

41
borgmatic/hooks/mysql.py
View File

@ -12,13 +12,11 @@ from borgmatic.hooks import dump
logger = logging.getLogger(__name__)
def make_dump_path(location_config): # pragma: no cover
def make_dump_path(config): # pragma: no cover
'''
Make the dump path from the given location configuration and the name of this hook.
Make the dump path from the given configuration dict and the name of this hook.
'''
return dump.make_database_dump_path(
location_config.get('borgmatic_source_directory'), 'mysql_databases'
)
return dump.make_database_dump_path(config.get('borgmatic_source_directory'), 'mysql_databases')
SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
@ -106,12 +104,11 @@ def execute_dump_command(
)
def dump_databases(databases, log_prefix, location_config, dry_run):
def dump_databases(databases, config, log_prefix, dry_run):
'''
Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence
of dicts, one dict describing each database as per the configuration schema. Use the given log
prefix in any log entries. Use the given location configuration dict to construct the
destination path.
of dicts, one dict describing each database as per the configuration schema. Use the given
configuration dict to construct the destination path and the given log prefix in any log entries.
Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
@ -122,7 +119,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
logger.info(f'{log_prefix}: Dumping MySQL databases{dry_run_label}')
for database in databases:
dump_path = make_dump_path(location_config)
dump_path = make_dump_path(config)
extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
dump_database_names = database_names_to_dump(
database, extra_environment, log_prefix, dry_run
@ -165,28 +162,26 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
return [process for process in processes if process]
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
'''
Remove all database dump files for this hook regardless of the given databases. Use the log
prefix in any log entries. Use the given location configuration dict to construct the
destination path. If this is a dry run, then don't actually remove anything.
Remove all database dump files for this hook regardless of the given databases. Use the given
configuration dict to construct the destination path and the log prefix in any log entries. If
this is a dry run, then don't actually remove anything.
'''
dump.remove_database_dumps(make_dump_path(location_config), 'MySQL', log_prefix, dry_run)
dump.remove_database_dumps(make_dump_path(config), 'MySQL', log_prefix, dry_run)
def make_database_dump_pattern(
databases, log_prefix, location_config, name=None
): # pragma: no cover
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
'''
Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
and a database name to match, return the corresponding glob patterns to match the database dump
in an archive.
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
database name to match, return the corresponding glob patterns to match the database dump in an
archive.
'''
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*')
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
database_config, config, log_prefix, dry_run, extract_process, connection_params
):
'''
Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a

2
borgmatic/hooks/ntfy.py
View File

@ -14,7 +14,7 @@ def initialize_monitor(
pass
def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
'''
Ping the configured Ntfy topic. Use the given configuration filename in any log entries.
If this is a dry run, then don't actually ping anything.

2
borgmatic/hooks/pagerduty.py
View File

@ -21,7 +21,7 @@ def initialize_monitor(
pass
def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
'''
If this is an error state, create a PagerDuty event with the configured integration key. Use
the given configuration filename in any log entries. If this is a dry run, then don't actually

48
borgmatic/hooks/postgresql.py
View File

@ -14,12 +14,12 @@ from borgmatic.hooks import dump
logger = logging.getLogger(__name__)
def make_dump_path(location_config): # pragma: no cover
def make_dump_path(config): # pragma: no cover
'''
Make the dump path from the given location configuration and the name of this hook.
Make the dump path from the given configuration dict and the name of this hook.
'''
return dump.make_database_dump_path(
location_config.get('borgmatic_source_directory'), 'postgresql_databases'
config.get('borgmatic_source_directory'), 'postgresql_databases'
)
@ -92,12 +92,12 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
)
def dump_databases(databases, log_prefix, location_config, dry_run):
def dump_databases(databases, config, log_prefix, dry_run):
'''
Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
dicts, one dict describing each database as per the configuration schema. Use the given log
prefix in any log entries. Use the given location configuration dict to construct the
destination path.
dicts, one dict describing each database as per the configuration schema. Use the given
configuration dict to construct the destination path and the given log prefix in any log
entries.
Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
@ -111,7 +111,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
for database in databases:
extra_environment = make_extra_environment(database)
dump_path = make_dump_path(location_config)
dump_path = make_dump_path(config)
dump_database_names = database_names_to_dump(
database, extra_environment, log_prefix, dry_run
)
@ -183,35 +183,33 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
return processes
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
'''
Remove all database dump files for this hook regardless of the given databases. Use the log
prefix in any log entries. Use the given location configuration dict to construct the
destination path. If this is a dry run, then don't actually remove anything.
Remove all database dump files for this hook regardless of the given databases. Use the given
configuration dict to construct the destination path and the log prefix in any log entries. If
this is a dry run, then don't actually remove anything.
'''
dump.remove_database_dumps(make_dump_path(location_config), 'PostgreSQL', log_prefix, dry_run)
dump.remove_database_dumps(make_dump_path(config), 'PostgreSQL', log_prefix, dry_run)
def make_database_dump_pattern(
databases, log_prefix, location_config, name=None
): # pragma: no cover
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
'''
Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
and a database name to match, return the corresponding glob patterns to match the database dump
in an archive.
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
database name to match, return the corresponding glob patterns to match the database dump in an
archive.
'''
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*')
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
database_config, config, log_prefix, dry_run, extract_process, connection_params
):
'''
Restore the given PostgreSQL database from an extract stream. The database is supplied as a
one-element sequence containing a dict describing the database, as per the configuration schema.
Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
output to consume.
Use the given configuration dict to construct the destination path and the given log prefix in
any log entries. If this is a dry run, then don't actually restore anything. Trigger the given
active extract process (an instance of subprocess.Popen) to produce output to consume.
If the extract process is None, then restore the dump from the filesystem rather than from an
extract stream.
@ -236,7 +234,7 @@ def restore_database_dump(
all_databases = bool(database['name'] == 'all')
dump_filename = dump.make_database_dump_filename(
make_dump_path(location_config), database['name'], database.get('hostname')
make_dump_path(config), database['name'], database.get('hostname')
)
psql_command = shlex.split(database.get('psql_command') or 'psql')
analyze_command = (

34
borgmatic/hooks/sqlite.py
View File

@ -7,21 +7,21 @@ from borgmatic.hooks import dump
logger = logging.getLogger(__name__)
def make_dump_path(location_config): # pragma: no cover
def make_dump_path(config): # pragma: no cover
'''
Make the dump path from the given location configuration and the name of this hook.
Make the dump path from the given configuration dict and the name of this hook.
'''
return dump.make_database_dump_path(
location_config.get('borgmatic_source_directory'), 'sqlite_databases'
config.get('borgmatic_source_directory'), 'sqlite_databases'
)
def dump_databases(databases, log_prefix, location_config, dry_run):
def dump_databases(databases, config, log_prefix, dry_run):
'''
Dump the given SQLite3 databases to a file. The databases are supplied as a sequence of
configuration dicts, as per the configuration schema. Use the given log prefix in any log
entries. Use the given location configuration dict to construct the destination path. If this
is a dry run, then don't actually dump anything.
configuration dicts, as per the configuration schema. Use the given configuration dict to
construct the destination path and the given log prefix in any log entries. If this is a dry
run, then don't actually dump anything.
'''
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
processes = []
@ -38,7 +38,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
f'{log_prefix}: No SQLite database at {database_path}; An empty database will be created and dumped'
)
dump_path = make_dump_path(location_config)
dump_path = make_dump_path(config)
dump_filename = dump.make_database_dump_filename(dump_path, database['name'])
if os.path.exists(dump_filename):
logger.warning(
@ -65,28 +65,26 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
return processes
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
def remove_database_dumps(databases, config, log_prefix, dry_run): # pragma: no cover
'''
Remove the given SQLite3 database dumps from the filesystem. The databases are supplied as a
sequence of configuration dicts, as per the configuration schema. Use the given log prefix in
any log entries. Use the given location configuration dict to construct the destination path.
If this is a dry run, then don't actually remove anything.
sequence of configuration dicts, as per the configuration schema. Use the given configuration
dict to construct the destination path and the given log prefix in any log entries. If this is a
dry run, then don't actually remove anything.
'''
dump.remove_database_dumps(make_dump_path(location_config), 'SQLite', log_prefix, dry_run)
dump.remove_database_dumps(make_dump_path(config), 'SQLite', log_prefix, dry_run)
def make_database_dump_pattern(
databases, log_prefix, location_config, name=None
): # pragma: no cover
def make_database_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
'''
Make a pattern that matches the given SQLite3 databases. The databases are supplied as a
sequence of configuration dicts, as per the configuration schema.
'''
return dump.make_database_dump_filename(make_dump_path(location_config), name)
return dump.make_database_dump_filename(make_dump_path(config), name)
def restore_database_dump(
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
database_config, config, log_prefix, dry_run, extract_process, connection_params
):
'''
Restore the given SQLite3 database from an extract stream. The database is supplied as a

1
tests/integration/borg/test_commands.py
View File

@ -84,7 +84,6 @@ def test_prune_archives_command_does_not_duplicate_flags_or_raise():
False,
'repo',
{},
{},
'2.3.4',
fuzz_argument(arguments, argument_name),
argparse.Namespace(log_json=False),

28
tests/integration/config/test_generate.py
View File

@ -13,43 +13,43 @@ def test_insert_newline_before_comment_does_not_raise():
config = module.yaml.comments.CommentedMap([(field_name, 33)])
config.yaml_set_comment_before_after_key(key=field_name, before='Comment')
module._insert_newline_before_comment(config, field_name)
module.insert_newline_before_comment(config, field_name)
def test_comment_out_line_skips_blank_line():
line = ' \n'
assert module._comment_out_line(line) == line
assert module.comment_out_line(line) == line
def test_comment_out_line_skips_already_commented_out_line():
line = ' # foo'
assert module._comment_out_line(line) == line
assert module.comment_out_line(line) == line
def test_comment_out_line_comments_section_name():
line = 'figgy-pudding:'
assert module._comment_out_line(line) == '# ' + line
assert module.comment_out_line(line) == '# ' + line
def test_comment_out_line_comments_indented_option():
line = ' enabled: true'
assert module._comment_out_line(line) == ' # enabled: true'
assert module.comment_out_line(line) == ' # enabled: true'
def test_comment_out_line_comments_twice_indented_option():
line = ' - item'
assert module._comment_out_line(line) == ' # - item'
assert module.comment_out_line(line) == ' # - item'
def test_comment_out_optional_configuration_comments_optional_config_only():
# The "# COMMENT_OUT" comment is a sentinel used to express that the following key is optional.
# It's stripped out of the final output.
flexmock(module)._comment_out_line = lambda line: '# ' + line
flexmock(module).comment_out_line = lambda line: '# ' + line
config = '''
# COMMENT_OUT
foo:
@ -84,7 +84,7 @@ location:
# other: thing
'''
assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()
assert module.comment_out_optional_configuration(config.strip()) == expected_config.strip()
def test_render_configuration_converts_configuration_to_yaml_string():
@ -204,10 +204,10 @@ def test_generate_sample_configuration_does_not_raise():
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('schema.yaml').and_return('')
flexmock(module.yaml).should_receive('round_trip_load')
flexmock(module).should_receive('_schema_to_sample_configuration')
flexmock(module).should_receive('schema_to_sample_configuration')
flexmock(module).should_receive('merge_source_configuration_into_destination')
flexmock(module).should_receive('render_configuration')
flexmock(module).should_receive('_comment_out_optional_configuration')
flexmock(module).should_receive('comment_out_optional_configuration')
flexmock(module).should_receive('write_configuration')
module.generate_sample_configuration(False, None, 'dest.yaml', 'schema.yaml')
@ -219,10 +219,10 @@ def test_generate_sample_configuration_with_source_filename_does_not_raise():
flexmock(module.yaml).should_receive('round_trip_load')
flexmock(module.load).should_receive('load_configuration')
flexmock(module.normalize).should_receive('normalize')
flexmock(module).should_receive('_schema_to_sample_configuration')
flexmock(module).should_receive('schema_to_sample_configuration')
flexmock(module).should_receive('merge_source_configuration_into_destination')
flexmock(module).should_receive('render_configuration')
flexmock(module).should_receive('_comment_out_optional_configuration')
flexmock(module).should_receive('comment_out_optional_configuration')
flexmock(module).should_receive('write_configuration')
module.generate_sample_configuration(False, 'source.yaml', 'dest.yaml', 'schema.yaml')
@ -232,10 +232,10 @@ def test_generate_sample_configuration_with_dry_run_does_not_write_file():
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('schema.yaml').and_return('')
flexmock(module.yaml).should_receive('round_trip_load')
flexmock(module).should_receive('_schema_to_sample_configuration')
flexmock(module).should_receive('schema_to_sample_configuration')
flexmock(module).should_receive('merge_source_configuration_into_destination')
flexmock(module).should_receive('render_configuration')
flexmock(module).should_receive('_comment_out_optional_configuration')
flexmock(module).should_receive('comment_out_optional_configuration')
flexmock(module).should_receive('write_configuration').never()
module.generate_sample_configuration(True, None, 'dest.yaml', 'schema.yaml')

149
tests/integration/config/test_validate.py
View File

@ -40,35 +40,32 @@ def mock_config_and_schema(config_yaml, schema_yaml=None):
def test_parse_configuration_transforms_file_into_mapping():
mock_config_and_schema(
'''
location:
source_directories:
- /home
- /etc
source_directories:
- /home
- /etc
repositories:
- path: hostname.borg
repositories:
- path: hostname.borg
retention:
keep_minutely: 60
keep_hourly: 24
keep_daily: 7
keep_minutely: 60
keep_hourly: 24
keep_daily: 7
consistency:
checks:
- name: repository
- name: archives
checks:
- name: repository
- name: archives
'''
)
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
assert config == {
'location': {
'source_directories': ['/home', '/etc'],
'repositories': [{'path': 'hostname.borg'}],
},
'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60},
'consistency': {'checks': [{'name': 'repository'}, {'name': 'archives'}]},
'source_directories': ['/home', '/etc'],
'repositories': [{'path': 'hostname.borg'}],
'keep_daily': 7,
'keep_hourly': 24,
'keep_minutely': 60,
'checks': [{'name': 'repository'}, {'name': 'archives'}],
}
assert logs == []
@ -78,22 +75,19 @@ def test_parse_configuration_passes_through_quoted_punctuation():
mock_config_and_schema(
f'''
location:
source_directories:
- "/home/{escaped_punctuation}"
source_directories:
- "/home/{escaped_punctuation}"
repositories:
- path: test.borg
repositories:
- path: test.borg
'''
)
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
assert config == {
'location': {
'source_directories': [f'/home/{string.punctuation}'],
'repositories': [{'path': 'test.borg'}],
}
'source_directories': [f'/home/{string.punctuation}'],
'repositories': [{'path': 'test.borg'}],
}
assert logs == []
@ -101,26 +95,22 @@ def test_parse_configuration_passes_through_quoted_punctuation():
def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
mock_config_and_schema(
'''
location:
source_directories:
- /home
source_directories:
- /home
repositories:
- path: hostname.borg
repositories:
- path: hostname.borg
''',
'''
map:
location:
source_directories:
required: true
map:
source_directories:
required: true
seq:
- type: scalar
repositories:
required: true
seq:
- type: scalar
seq:
- type: scalar
repositories:
required: true
seq:
- type: scalar
''',
)
@ -130,12 +120,11 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
def test_parse_configuration_inlines_include():
mock_config_and_schema(
'''
location:
source_directories:
- /home
source_directories:
- /home
repositories:
- path: hostname.borg
repositories:
- path: hostname.borg
retention:
!include include.yaml
@ -154,25 +143,25 @@ def test_parse_configuration_inlines_include():
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
assert config == {
'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]},
'retention': {'keep_daily': 7, 'keep_hourly': 24},
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg'}],
'keep_daily': 7,
'keep_hourly': 24,
}
assert logs == []
assert len(logs) == 1
def test_parse_configuration_merges_include():
mock_config_and_schema(
'''
location:
source_directories:
- /home
source_directories:
- /home
repositories:
- path: hostname.borg
repositories:
- path: hostname.borg
retention:
keep_daily: 1
<<: !include include.yaml
keep_daily: 1
<<: !include include.yaml
'''
)
builtins = flexmock(sys.modules['builtins'])
@ -188,8 +177,10 @@ def test_parse_configuration_merges_include():
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
assert config == {
'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]},
'retention': {'keep_daily': 1, 'keep_hourly': 24},
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg'}],
'keep_daily': 1,
'keep_hourly': 24,
}
assert logs == []
@ -218,10 +209,9 @@ def test_parse_configuration_raises_for_syntax_error():
def test_parse_configuration_raises_for_validation_error():
mock_config_and_schema(
'''
location:
source_directories: yes
repositories:
- path: hostname.borg
source_directories: yes
repositories:
- path: hostname.borg
'''
)
@ -232,14 +222,13 @@ def test_parse_configuration_raises_for_validation_error():
def test_parse_configuration_applies_overrides():
mock_config_and_schema(
'''
location:
source_directories:
- /home
source_directories:
- /home
repositories:
- path: hostname.borg
repositories:
- path: hostname.borg
local_path: borg1
local_path: borg1
'''
)
@ -248,11 +237,9 @@ def test_parse_configuration_applies_overrides():
)
assert config == {
'location': {
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg'}],
'local_path': 'borg2',
}
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg'}],
'local_path': 'borg2',
}
assert logs == []
@ -274,10 +261,8 @@ def test_parse_configuration_applies_normalization():
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
assert config == {
'location': {
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg'}],
'exclude_if_present': ['.nobackup'],
}
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg'}],
'exclude_if_present': ['.nobackup'],
}
assert logs

2
tests/unit/actions/test_borg.py
View File

@ -14,7 +14,7 @@ def test_run_borg_does_not_raise():
module.run_borg(
repository={'path': 'repos'},
storage={},
config={},
local_borg_version=None,
global_arguments=flexmock(log_json=False),
borg_arguments=borg_arguments,

2
tests/unit/actions/test_break_lock.py
View File

@ -11,7 +11,7 @@ def test_run_break_lock_does_not_raise():
module.run_break_lock(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
break_lock_arguments=break_lock_arguments,
global_arguments=flexmock(),

18
tests/unit/actions/test_check.py
View File

@ -5,9 +5,6 @@ from borgmatic.actions import check as module
def test_run_check_calls_hooks_for_configured_repository():
flexmock(module.logger).answer = lambda message: None
flexmock(module.borgmatic.config.checks).should_receive(
'repository_enabled_for_checks'
).and_return(True)
flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
flexmock(module.borgmatic.borg.check).should_receive('check_archives').once()
flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
@ -23,10 +20,7 @@ def test_run_check_calls_hooks_for_configured_repository():
module.run_check(
config_filename='test.yaml',
repository={'path': 'repo'},
location={'repositories': ['repo']},
storage={},
consistency={},
hooks={},
config={'repositories': ['repo']},
hook_context={},
local_borg_version=None,
check_arguments=check_arguments,
@ -54,10 +48,7 @@ def test_run_check_runs_with_selected_repository():
module.run_check(
config_filename='test.yaml',
repository={'path': 'repo'},
location={'repositories': ['repo']},
storage={},
consistency={},
hooks={},
config={'repositories': ['repo']},
hook_context={},
local_borg_version=None,
check_arguments=check_arguments,
@ -85,10 +76,7 @@ def test_run_check_bails_if_repository_does_not_match():
module.run_check(
config_filename='test.yaml',
repository={'path': 'repo'},
location={'repositories': ['repo']},
storage={},
consistency={},
hooks={},
config={'repositories': ['repo']},
hook_context={},
local_borg_version=None,
check_arguments=check_arguments,

12
tests/unit/actions/test_compact.py
View File

@ -17,9 +17,7 @@ def test_compact_actions_calls_hooks_for_configured_repository():
module.run_compact(
config_filename='test.yaml',
repository={'path': 'repo'},
storage={},
retention={},
hooks={},
config={},
hook_context={},
local_borg_version=None,
compact_arguments=compact_arguments,
@ -45,9 +43,7 @@ def test_compact_runs_with_selected_repository():
module.run_compact(
config_filename='test.yaml',
repository={'path': 'repo'},
storage={},
retention={},
hooks={},
config={},
hook_context={},
local_borg_version=None,
compact_arguments=compact_arguments,
@ -73,9 +69,7 @@ def test_compact_bails_if_repository_does_not_match():
module.run_compact(
config_filename='test.yaml',
repository={'path': 'repo'},
storage={},
retention={},
hooks={},
config={},
hook_context={},
local_borg_version=None,
compact_arguments=compact_arguments,

17
tests/unit/actions/test_create.py
View File

@ -28,9 +28,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
module.run_create(
config_filename='test.yaml',
repository={'path': 'repo'},
location={},
storage={},
hooks={},
config={},
hook_context={},
local_borg_version=None,
create_arguments=create_arguments,
@ -49,6 +47,11 @@ def test_run_create_runs_with_selected_repository():
).once().and_return(True)
flexmock(module.borgmatic.borg.create).should_receive('create_archive').once()
flexmock(module).should_receive('create_borgmatic_manifest').once()
flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({})
flexmock(module.borgmatic.hooks.dispatch).should_receive(
'call_hooks_even_if_unconfigured'
).and_return({})
create_arguments = flexmock(
repository=flexmock(),
progress=flexmock(),
@ -62,9 +65,7 @@ def test_run_create_runs_with_selected_repository():
module.run_create(
config_filename='test.yaml',
repository={'path': 'repo'},
location={},
storage={},
hooks={},
config={},
hook_context={},
local_borg_version=None,
create_arguments=create_arguments,
@ -96,9 +97,7 @@ def test_run_create_bails_if_repository_does_not_match():
module.run_create(
config_filename='test.yaml',
repository='repo',
location={},
storage={},
hooks={},
config={},
hook_context={},
local_borg_version=None,
create_arguments=create_arguments,

2
tests/unit/actions/test_export_tar.py
View File

@ -20,7 +20,7 @@ def test_run_export_tar_does_not_raise():
module.run_export_tar(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
export_tar_arguments=export_tar_arguments,
global_arguments=global_arguments,

4
tests/unit/actions/test_extract.py
View File

@ -21,9 +21,7 @@ def test_run_extract_calls_hooks():
module.run_extract(
config_filename='test.yaml',
repository={'path': 'repo'},
location={'repositories': ['repo']},
storage={},
hooks={},
config={'repositories': ['repo']},
hook_context={},
local_borg_version=None,
extract_arguments=extract_arguments,

2
tests/unit/actions/test_info.py
View File

@ -18,7 +18,7 @@ def test_run_info_does_not_raise():
list(
module.run_info(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
info_arguments=info_arguments,
global_arguments=flexmock(log_json=False),

2
tests/unit/actions/test_list.py
View File

@ -18,7 +18,7 @@ def test_run_list_does_not_raise():
list(
module.run_list(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
list_arguments=list_arguments,
global_arguments=flexmock(log_json=False),

2
tests/unit/actions/test_mount.py
View File

@ -18,7 +18,7 @@ def test_run_mount_does_not_raise():
module.run_mount(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
mount_arguments=mount_arguments,
global_arguments=flexmock(log_json=False),

12
tests/unit/actions/test_prune.py
View File

@ -14,9 +14,7 @@ def test_run_prune_calls_hooks_for_configured_repository():
module.run_prune(
config_filename='test.yaml',
repository={'path': 'repo'},
storage={},
retention={},
hooks={},
config={},
hook_context={},
local_borg_version=None,
prune_arguments=prune_arguments,
@ -39,9 +37,7 @@ def test_run_prune_runs_with_selected_repository():
module.run_prune(
config_filename='test.yaml',
repository={'path': 'repo'},
storage={},
retention={},
hooks={},
config={},
hook_context={},
local_borg_version=None,
prune_arguments=prune_arguments,
@ -64,9 +60,7 @@ def test_run_prune_bails_if_repository_does_not_match():
module.run_prune(
config_filename='test.yaml',
repository='repo',
storage={},
retention={},
hooks={},
config={},
hook_context={},
local_borg_version=None,
prune_arguments=prune_arguments,

4
tests/unit/actions/test_rcreate.py
View File

@ -19,7 +19,7 @@ def test_run_rcreate_does_not_raise():
module.run_rcreate(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
rcreate_arguments=arguments,
global_arguments=flexmock(dry_run=False),
@ -46,7 +46,7 @@ def test_run_rcreate_bails_if_repository_does_not_match():
module.run_rcreate(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
rcreate_arguments=arguments,
global_arguments=flexmock(dry_run=False),

87
tests/unit/actions/test_restore.py
View File

@ -6,7 +6,7 @@ import borgmatic.actions.restore as module
def test_get_configured_database_matches_database_by_name():
assert module.get_configured_database(
hooks={
config={
'other_databases': [{'name': 'other'}],
'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}],
},
@ -18,7 +18,7 @@ def test_get_configured_database_matches_database_by_name():
def test_get_configured_database_matches_nothing_when_database_name_not_configured():
assert module.get_configured_database(
hooks={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]},
config={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]},
archive_database_names={'postgresql_databases': ['foo']},
hook_name='postgresql_databases',
database_name='quux',
@ -27,7 +27,7 @@ def test_get_configured_database_matches_nothing_when_database_name_not_configur
def test_get_configured_database_matches_nothing_when_database_name_not_in_archive():
assert module.get_configured_database(
hooks={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]},
config={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]},
archive_database_names={'postgresql_databases': ['bar']},
hook_name='postgresql_databases',
database_name='foo',
@ -36,7 +36,7 @@ def test_get_configured_database_matches_nothing_when_database_name_not_in_archi
def test_get_configured_database_matches_database_by_configuration_database_name():
assert module.get_configured_database(
hooks={'postgresql_databases': [{'name': 'all'}, {'name': 'bar'}]},
config={'postgresql_databases': [{'name': 'all'}, {'name': 'bar'}]},
archive_database_names={'postgresql_databases': ['foo']},
hook_name='postgresql_databases',
database_name='foo',
@ -46,7 +46,7 @@ def test_get_configured_database_matches_database_by_configuration_database_name
def test_get_configured_database_with_unspecified_hook_matches_database_by_name():
assert module.get_configured_database(
hooks={
config={
'other_databases': [{'name': 'other'}],
'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}],
},
@ -69,8 +69,7 @@ def test_collect_archive_database_names_parses_archive_paths():
archive_database_names = module.collect_archive_database_names(
repository={'path': 'repo'},
archive='archive',
location={'borgmatic_source_directory': '.borgmatic'},
storage=flexmock(),
config={'borgmatic_source_directory': '.borgmatic'},
local_borg_version=flexmock(),
global_arguments=flexmock(log_json=False),
local_path=flexmock(),
@ -95,8 +94,7 @@ def test_collect_archive_database_names_parses_directory_format_archive_paths():
archive_database_names = module.collect_archive_database_names(
repository={'path': 'repo'},
archive='archive',
location={'borgmatic_source_directory': '.borgmatic'},
storage=flexmock(),
config={'borgmatic_source_directory': '.borgmatic'},
local_borg_version=flexmock(),
global_arguments=flexmock(log_json=False),
local_path=flexmock(),
@ -117,8 +115,7 @@ def test_collect_archive_database_names_skips_bad_archive_paths():
archive_database_names = module.collect_archive_database_names(
repository={'path': 'repo'},
archive='archive',
location={'borgmatic_source_directory': '.borgmatic'},
storage=flexmock(),
config={'borgmatic_source_directory': '.borgmatic'},
local_borg_version=flexmock(),
global_arguments=flexmock(log_json=False),
local_path=flexmock(),
@ -231,9 +228,7 @@ def test_run_restore_restores_each_database():
).and_return(('postgresql_databases', {'name': 'bar'}))
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
location=object,
storage=object,
hooks=object,
config=object,
local_borg_version=object,
global_arguments=object,
local_path=object,
@ -245,9 +240,7 @@ def test_run_restore_restores_each_database():
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
location=object,
storage=object,
hooks=object,
config=object,
local_borg_version=object,
global_arguments=object,
local_path=object,
@ -261,9 +254,7 @@ def test_run_restore_restores_each_database():
module.run_restore(
repository={'path': 'repo'},
location=flexmock(),
storage=flexmock(),
hooks=flexmock(),
config=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo',
@ -293,9 +284,7 @@ def test_run_restore_bails_for_non_matching_repository():
module.run_restore(
repository={'path': 'repo'},
location=flexmock(),
storage=flexmock(),
hooks=flexmock(),
config=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()),
global_arguments=flexmock(dry_run=False),
@ -317,19 +306,19 @@ def test_run_restore_restores_database_configured_with_all_name():
flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock())
flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names)
flexmock(module).should_receive('get_configured_database').with_args(
hooks=object,
config=object,
archive_database_names=object,
hook_name='postgresql_databases',
database_name='foo',
).and_return(('postgresql_databases', {'name': 'foo'}))
flexmock(module).should_receive('get_configured_database').with_args(
hooks=object,
config=object,
archive_database_names=object,
hook_name='postgresql_databases',
database_name='bar',
).and_return((None, None))
flexmock(module).should_receive('get_configured_database').with_args(
hooks=object,
config=object,
archive_database_names=object,
hook_name='postgresql_databases',
database_name='bar',
@ -337,9 +326,7 @@ def test_run_restore_restores_database_configured_with_all_name():
).and_return(('postgresql_databases', {'name': 'bar'}))
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
location=object,
storage=object,
hooks=object,
config=object,
local_borg_version=object,
global_arguments=object,
local_path=object,
@ -351,9 +338,7 @@ def test_run_restore_restores_database_configured_with_all_name():
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
location=object,
storage=object,
hooks=object,
config=object,
local_borg_version=object,
global_arguments=object,
local_path=object,
@ -367,9 +352,7 @@ def test_run_restore_restores_database_configured_with_all_name():
module.run_restore(
repository={'path': 'repo'},
location=flexmock(),
storage=flexmock(),
hooks=flexmock(),
config=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo',
@ -401,19 +384,19 @@ def test_run_restore_skips_missing_database():
flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock())
flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names)
flexmock(module).should_receive('get_configured_database').with_args(
hooks=object,
config=object,
archive_database_names=object,
hook_name='postgresql_databases',
database_name='foo',
).and_return(('postgresql_databases', {'name': 'foo'}))
flexmock(module).should_receive('get_configured_database').with_args(
hooks=object,
config=object,
archive_database_names=object,
hook_name='postgresql_databases',
database_name='bar',
).and_return((None, None))
flexmock(module).should_receive('get_configured_database').with_args(
hooks=object,
config=object,
archive_database_names=object,
hook_name='postgresql_databases',
database_name='bar',
@ -421,9 +404,7 @@ def test_run_restore_skips_missing_database():
).and_return((None, None))
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
location=object,
storage=object,
hooks=object,
config=object,
local_borg_version=object,
global_arguments=object,
local_path=object,
@ -435,9 +416,7 @@ def test_run_restore_skips_missing_database():
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
location=object,
storage=object,
hooks=object,
config=object,
local_borg_version=object,
global_arguments=object,
local_path=object,
@ -451,9 +430,7 @@ def test_run_restore_skips_missing_database():
module.run_restore(
repository={'path': 'repo'},
location=flexmock(),
storage=flexmock(),
hooks=flexmock(),
config=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo',
@ -486,22 +463,20 @@ def test_run_restore_restores_databases_from_different_hooks():
flexmock(module).should_receive('collect_archive_database_names').and_return(flexmock())
flexmock(module).should_receive('find_databases_to_restore').and_return(restore_names)
flexmock(module).should_receive('get_configured_database').with_args(
hooks=object,
config=object,
archive_database_names=object,
hook_name='postgresql_databases',
database_name='foo',
).and_return(('postgresql_databases', {'name': 'foo'}))
flexmock(module).should_receive('get_configured_database').with_args(
hooks=object,
config=object,
archive_database_names=object,
hook_name='mysql_databases',
database_name='bar',
).and_return(('mysql_databases', {'name': 'bar'}))
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
location=object,
storage=object,
hooks=object,
config=object,
local_borg_version=object,
global_arguments=object,
local_path=object,
@ -513,9 +488,7 @@ def test_run_restore_restores_databases_from_different_hooks():
).once()
flexmock(module).should_receive('restore_single_database').with_args(
repository=object,
location=object,
storage=object,
hooks=object,
config=object,
local_borg_version=object,
global_arguments=object,
local_path=object,
@ -529,9 +502,7 @@ def test_run_restore_restores_databases_from_different_hooks():
module.run_restore(
repository={'path': 'repo'},
location=flexmock(),
storage=flexmock(),
hooks=flexmock(),
config=flexmock(),
local_borg_version=flexmock(),
restore_arguments=flexmock(
repository='repo',

2
tests/unit/actions/test_rinfo.py
View File

@ -12,7 +12,7 @@ def test_run_rinfo_does_not_raise():
list(
module.run_rinfo(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
rinfo_arguments=rinfo_arguments,
global_arguments=flexmock(log_json=False),

2
tests/unit/actions/test_rlist.py
View File

@ -12,7 +12,7 @@ def test_run_rlist_does_not_raise():
list(
module.run_rlist(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
rlist_arguments=rlist_arguments,
global_arguments=flexmock(),

2
tests/unit/actions/test_transfer.py
View File

@ -11,7 +11,7 @@ def test_run_transfer_does_not_raise():
module.run_transfer(
repository={'path': 'repo'},
storage={},
config={},
local_borg_version=None,
transfer_arguments=transfer_arguments,
global_arguments=global_arguments,

26
tests/unit/borg/test_borg.py
View File

@ -22,7 +22,7 @@ def test_run_arbitrary_borg_calls_borg_with_flags():
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['break-lock', '::'],
)
@ -44,7 +44,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag():
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['break-lock', '::'],
)
@ -66,7 +66,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag():
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['break-lock', '::'],
)
@ -75,7 +75,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag():
def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags():
flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
storage_config = {'lock_wait': 5}
config = {'lock_wait': 5}
flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(
('--lock-wait', '5')
)
@ -90,7 +90,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags():
module.run_arbitrary_borg(
repository_path='repo',
storage_config=storage_config,
config=config,
local_borg_version='1.2.3',
options=['break-lock', '::'],
)
@ -111,7 +111,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag():
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['break-lock', '::$ARCHIVE'],
archive='archive',
@ -133,7 +133,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['break-lock', '::'],
local_path='borg1',
@ -157,7 +157,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags()
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['break-lock', '::'],
remote_path='borg1',
@ -179,7 +179,7 @@ def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg():
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['list', '--progress', '::'],
)
@ -200,7 +200,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg():
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['--', 'break-lock', '::'],
)
@ -221,7 +221,7 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise():
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=[],
)
@ -243,7 +243,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['key', 'export', '::'],
)
@ -265,7 +265,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla
module.run_arbitrary_borg(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
options=['debug', 'dump-manifest', '::', 'path'],
)

14
tests/unit/borg/test_break_lock.py
View File

@ -22,7 +22,7 @@ def test_break_lock_calls_borg_with_required_flags():
module.break_lock(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -34,7 +34,7 @@ def test_break_lock_calls_borg_with_remote_path_flags():
module.break_lock(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
remote_path='borg1',
@ -47,7 +47,7 @@ def test_break_lock_calls_borg_with_umask_flags():
module.break_lock(
repository_path='repo',
storage_config={'umask': '0770'},
config={'umask': '0770'},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -59,7 +59,7 @@ def test_break_lock_calls_borg_with_log_json_flags():
module.break_lock(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=True),
)
@ -71,7 +71,7 @@ def test_break_lock_calls_borg_with_lock_wait_flags():
module.break_lock(
repository_path='repo',
storage_config={'lock_wait': '5'},
config={'lock_wait': '5'},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -84,7 +84,7 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter():
module.break_lock(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -97,7 +97,7 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags():
module.break_lock(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)

113
tests/unit/borg/test_check.py
View File

@ -99,8 +99,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks():
flexmock(module).should_receive('probe_for_check_time').and_return(None)
assert module.filter_checks_on_frequency(
location_config={},
consistency_config={},
config={},
borg_repository_id='repo',
checks=('repository', 'archives'),
force=False,
@ -110,8 +109,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks():
def test_filter_checks_on_frequency_retains_unconfigured_check():
assert module.filter_checks_on_frequency(
location_config={},
consistency_config={},
config={},
borg_repository_id='repo',
checks=('data',),
force=False,
@ -122,8 +120,7 @@ def test_filter_checks_on_frequency_retains_check_without_frequency():
flexmock(module).should_receive('parse_frequency').and_return(None)
assert module.filter_checks_on_frequency(
location_config={},
consistency_config={'checks': [{'name': 'archives'}]},
config={'checks': [{'name': 'archives'}]},
borg_repository_id='repo',
checks=('archives',),
force=False,
@ -141,8 +138,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency():
)
assert module.filter_checks_on_frequency(
location_config={},
consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
borg_repository_id='repo',
checks=('archives',),
force=False,
@ -158,8 +154,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file()
flexmock(module).should_receive('probe_for_check_time').and_return(None)
assert module.filter_checks_on_frequency(
location_config={},
consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
borg_repository_id='repo',
checks=('archives',),
force=False,
@ -178,8 +173,7 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency():
assert (
module.filter_checks_on_frequency(
location_config={},
consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
borg_repository_id='repo',
checks=('archives',),
force=False,
@ -191,8 +185,7 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency():
def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_force():
assert module.filter_checks_on_frequency(
location_config={},
consistency_config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
borg_repository_id='repo',
checks=('archives',),
force=True,
@ -616,7 +609,7 @@ def test_upgrade_check_times_renames_stale_temporary_check_path():
def test_check_archives_with_progress_calls_borg_with_progress_parameter():
checks = ('repository',)
consistency_config = {'check_last': None}
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -639,9 +632,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter():
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
progress=True,
@ -650,7 +641,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter():
def test_check_archives_with_repair_calls_borg_with_repair_parameter():
checks = ('repository',)
consistency_config = {'check_last': None}
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -673,9 +664,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter():
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
repair=True,
@ -693,7 +682,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter():
)
def test_check_archives_calls_borg_with_parameters(checks):
check_last = flexmock()
consistency_config = {'check_last': check_last}
config = {'check_last': check_last}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -710,9 +699,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -721,7 +708,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
def test_check_archives_with_json_error_raises():
checks = ('archives',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
config = {'check_last': check_last}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"unexpected": {"id": "repo"}}'
)
@ -734,9 +721,7 @@ def test_check_archives_with_json_error_raises():
with pytest.raises(ValueError):
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -745,7 +730,7 @@ def test_check_archives_with_json_error_raises():
def test_check_archives_with_missing_json_keys_raises():
checks = ('archives',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
config = {'check_last': check_last}
flexmock(module.rinfo).should_receive('display_repository_info').and_return('{invalid JSON')
flexmock(module).should_receive('upgrade_check_times')
flexmock(module).should_receive('parse_checks')
@ -756,9 +741,7 @@ def test_check_archives_with_missing_json_keys_raises():
with pytest.raises(ValueError):
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -767,7 +750,7 @@ def test_check_archives_with_missing_json_keys_raises():
def test_check_archives_with_extract_check_calls_extract_only():
checks = ('extract',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
config = {'check_last': check_last}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -784,9 +767,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -794,7 +775,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
def test_check_archives_with_log_info_calls_borg_with_info_parameter():
checks = ('repository',)
consistency_config = {'check_last': None}
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -812,9 +793,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter():
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -822,7 +801,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter():
def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
checks = ('repository',)
consistency_config = {'check_last': None}
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -840,16 +819,14 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
def test_check_archives_without_any_checks_bails():
consistency_config = {'check_last': None}
config = {'check_last': None}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -862,9 +839,7 @@ def test_check_archives_without_any_checks_bails():
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -873,7 +848,7 @@ def test_check_archives_without_any_checks_bails():
def test_check_archives_with_local_path_calls_borg_via_local_path():
checks = ('repository',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
config = {'check_last': check_last}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -890,9 +865,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
local_path='borg1',
@ -902,7 +875,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters():
checks = ('repository',)
check_last = flexmock()
consistency_config = {'check_last': check_last}
config = {'check_last': check_last}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -919,9 +892,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
remote_path='borg1',
@ -931,8 +902,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
def test_check_archives_with_log_json_calls_borg_with_log_json_parameters():
checks = ('repository',)
check_last = flexmock()
storage_config = {}
consistency_config = {'check_last': check_last}
config = {'check_last': check_last}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -949,9 +919,7 @@ def test_check_archives_with_log_json_calls_borg_with_log_json_parameters():
module.check_archives(
repository_path='repo',
location_config={},
storage_config=storage_config,
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=True),
)
@ -960,8 +928,7 @@ def test_check_archives_with_log_json_calls_borg_with_log_json_parameters():
def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
checks = ('repository',)
check_last = flexmock()
storage_config = {'lock_wait': 5}
consistency_config = {'check_last': check_last}
config = {'lock_wait': 5, 'check_last': check_last}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -978,9 +945,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
module.check_archives(
repository_path='repo',
location_config={},
storage_config=storage_config,
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -990,7 +955,7 @@ def test_check_archives_with_retention_prefix():
checks = ('repository',)
check_last = flexmock()
prefix = 'foo-'
consistency_config = {'check_last': check_last, 'prefix': prefix}
config = {'check_last': check_last, 'prefix': prefix}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -1007,9 +972,7 @@ def test_check_archives_with_retention_prefix():
module.check_archives(
repository_path='repo',
location_config={},
storage_config={},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -1017,7 +980,7 @@ def test_check_archives_with_retention_prefix():
def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options():
checks = ('repository',)
consistency_config = {'check_last': None}
config = {'check_last': None, 'extra_borg_options': {'check': '--extra --options'}}
flexmock(module.rinfo).should_receive('display_repository_info').and_return(
'{"repository": {"id": "repo"}}'
)
@ -1034,9 +997,7 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options():
module.check_archives(
repository_path='repo',
location_config={},
storage_config={'extra_borg_options': {'check': '--extra --options'}},
consistency_config=consistency_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)

30
tests/unit/borg/test_compact.py
View File

@ -27,7 +27,7 @@ def test_compact_segments_calls_borg_with_parameters():
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -40,7 +40,7 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter():
module.compact_segments(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
dry_run=False,
@ -54,7 +54,7 @@ def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter():
module.compact_segments(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
dry_run=False,
@ -66,7 +66,7 @@ def test_compact_segments_with_dry_run_skips_borg_call():
module.compact_segments(
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
dry_run=True,
@ -80,7 +80,7 @@ def test_compact_segments_with_local_path_calls_borg_via_local_path():
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
local_path='borg1',
@ -94,7 +94,7 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
remote_path='borg1',
@ -108,7 +108,7 @@ def test_compact_segments_with_progress_calls_borg_with_progress_parameter():
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
progress=True,
@ -122,7 +122,7 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
cleanup_commits=True,
@ -136,7 +136,7 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter():
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
threshold=20,
@ -144,14 +144,14 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter():
def test_compact_segments_with_umask_calls_borg_with_umask_parameters():
storage_config = {'umask': '077'}
config = {'umask': '077'}
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.INFO)
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config=storage_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -164,21 +164,21 @@ def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters():
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=True),
)
def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config = {'lock_wait': 5}
config = {'lock_wait': 5}
flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO)
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config=storage_config,
config=config,
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -191,7 +191,7 @@ def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options(
module.compact_segments(
dry_run=False,
repository_path='repo',
storage_config={'extra_borg_options': {'compact': '--extra --options'}},
config={'extra_borg_options': {'compact': '--extra --options'}},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)

191
tests/unit/borg/test_create.py
View File

@ -170,79 +170,75 @@ def test_ensure_files_readable_opens_filenames(filename_lists, opened_filenames)
def test_make_pattern_flags_includes_pattern_filename_when_given():
pattern_flags = module.make_pattern_flags(
location_config={'patterns': ['R /', '- /var']}, pattern_filename='/tmp/patterns'
config={'patterns': ['R /', '- /var']}, pattern_filename='/tmp/patterns'
)
assert pattern_flags == ('--patterns-from', '/tmp/patterns')
def test_make_pattern_flags_includes_patterns_from_filenames_when_in_config():
pattern_flags = module.make_pattern_flags(
location_config={'patterns_from': ['patterns', 'other']}
)
pattern_flags = module.make_pattern_flags(config={'patterns_from': ['patterns', 'other']})
assert pattern_flags == ('--patterns-from', 'patterns', '--patterns-from', 'other')
def test_make_pattern_flags_includes_both_filenames_when_patterns_given_and_patterns_from_in_config():
pattern_flags = module.make_pattern_flags(
location_config={'patterns_from': ['patterns']}, pattern_filename='/tmp/patterns'
config={'patterns_from': ['patterns']}, pattern_filename='/tmp/patterns'
)
assert pattern_flags == ('--patterns-from', 'patterns', '--patterns-from', '/tmp/patterns')
def test_make_pattern_flags_considers_none_patterns_from_filenames_as_empty():
pattern_flags = module.make_pattern_flags(location_config={'patterns_from': None})
pattern_flags = module.make_pattern_flags(config={'patterns_from': None})
assert pattern_flags == ()
def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
exclude_flags = module.make_exclude_flags(
location_config={'exclude_patterns': ['*.pyc', '/var']}, exclude_filename='/tmp/excludes'
config={'exclude_patterns': ['*.pyc', '/var']}, exclude_filename='/tmp/excludes'
)
assert exclude_flags == ('--exclude-from', '/tmp/excludes')
def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
exclude_flags = module.make_exclude_flags(
location_config={'exclude_from': ['excludes', 'other']}
)
exclude_flags = module.make_exclude_flags(config={'exclude_from': ['excludes', 'other']})
assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', 'other')
def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config():
exclude_flags = module.make_exclude_flags(
location_config={'exclude_from': ['excludes']}, exclude_filename='/tmp/excludes'
config={'exclude_from': ['excludes']}, exclude_filename='/tmp/excludes'
)
assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty():
exclude_flags = module.make_exclude_flags(location_config={'exclude_from': None})
exclude_flags = module.make_exclude_flags(config={'exclude_from': None})
assert exclude_flags == ()
def test_make_exclude_flags_includes_exclude_caches_when_true_in_config():
exclude_flags = module.make_exclude_flags(location_config={'exclude_caches': True})
exclude_flags = module.make_exclude_flags(config={'exclude_caches': True})
assert exclude_flags == ('--exclude-caches',)
def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config():
exclude_flags = module.make_exclude_flags(location_config={'exclude_caches': False})
exclude_flags = module.make_exclude_flags(config={'exclude_caches': False})
assert exclude_flags == ()
def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
exclude_flags = module.make_exclude_flags(
location_config={'exclude_if_present': ['exclude_me', 'also_me']}
config={'exclude_if_present': ['exclude_me', 'also_me']}
)
assert exclude_flags == (
@ -254,31 +250,31 @@ def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
def test_make_exclude_flags_includes_keep_exclude_tags_when_true_in_config():
exclude_flags = module.make_exclude_flags(location_config={'keep_exclude_tags': True})
exclude_flags = module.make_exclude_flags(config={'keep_exclude_tags': True})
assert exclude_flags == ('--keep-exclude-tags',)
def test_make_exclude_flags_does_not_include_keep_exclude_tags_when_false_in_config():
exclude_flags = module.make_exclude_flags(location_config={'keep_exclude_tags': False})
exclude_flags = module.make_exclude_flags(config={'keep_exclude_tags': False})
assert exclude_flags == ()
def test_make_exclude_flags_includes_exclude_nodump_when_true_in_config():
exclude_flags = module.make_exclude_flags(location_config={'exclude_nodump': True})
exclude_flags = module.make_exclude_flags(config={'exclude_nodump': True})
assert exclude_flags == ('--exclude-nodump',)
def test_make_exclude_flags_does_not_include_exclude_nodump_when_false_in_config():
exclude_flags = module.make_exclude_flags(location_config={'exclude_nodump': False})
exclude_flags = module.make_exclude_flags(config={'exclude_nodump': False})
assert exclude_flags == ()
def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
exclude_flags = module.make_exclude_flags(location_config={})
exclude_flags = module.make_exclude_flags(config={})
assert exclude_flags == ()
@ -504,12 +500,11 @@ def test_create_archive_calls_borg_with_parameters():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -548,12 +543,11 @@ def test_create_archive_calls_borg_with_environment():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -594,12 +588,11 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'patterns': ['pattern'],
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -644,11 +637,10 @@ def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sourc
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']),
)
@ -689,12 +681,11 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': ['exclude'],
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -733,12 +724,11 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -774,12 +764,11 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
json=True,
@ -819,12 +808,11 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -860,12 +848,11 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
json=True,
@ -904,12 +891,11 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
module.create_archive(
dry_run=True,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -950,12 +936,11 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
module.create_archive(
dry_run=True,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stats=True,
@ -994,12 +979,12 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'checkpoint_interval': 600,
},
storage_config={'checkpoint_interval': 600},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1037,12 +1022,12 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'checkpoint_volume': 1024,
},
storage_config={'checkpoint_volume': 1024},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1080,12 +1065,12 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'chunker_params': '1,2,3,4',
},
storage_config={'chunker_params': '1,2,3,4'},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1123,12 +1108,12 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'compression': 'rle',
},
storage_config={'compression': 'rle'},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1172,12 +1157,12 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'upload_rate_limit': 100,
},
storage_config={'upload_rate_limit': 100},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1217,13 +1202,12 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'working_directory': '/working/dir',
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1261,13 +1245,12 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'one_file_system': True,
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1311,13 +1294,12 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter(
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'numeric_ids': True,
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1365,13 +1347,12 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'read_special': True,
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1421,13 +1402,12 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
option_name: option_value,
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1476,13 +1456,12 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'atime': option_value,
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1531,13 +1510,12 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'flags': option_value,
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1575,13 +1553,12 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'files_cache': 'ctime,size',
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1619,12 +1596,11 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
local_path='borg1',
@ -1663,12 +1639,11 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
remote_path='borg1',
@ -1707,12 +1682,12 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'umask': 740,
},
storage_config={'umask': 740},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1750,12 +1725,11 @@ def test_create_archive_with_log_json_calls_borg_with_log_json_parameters():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=True, used_config_paths=[]),
)
@ -1793,12 +1767,12 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'lock_wait': 5,
},
storage_config={'lock_wait': 5},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -1836,12 +1810,11 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stats=True,
@ -1880,12 +1853,11 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
list_files=True,
@ -1930,12 +1902,11 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
progress=True,
@ -1974,12 +1945,11 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
progress=True,
@ -2035,12 +2005,11 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
progress=True,
@ -2099,13 +2068,12 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'read_special': False,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stream_processes=processes,
@ -2168,12 +2136,11 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stream_processes=processes,
@ -2232,13 +2199,12 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'read_special': True,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stream_processes=processes,
@ -2274,12 +2240,11 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
json_output = module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
json=True,
@ -2317,12 +2282,11 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
json_output = module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
json=True,
@ -2365,12 +2329,11 @@ def test_create_archive_with_source_directories_glob_expands():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo*'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2409,12 +2372,11 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo*'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2452,12 +2414,11 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo*'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2495,12 +2456,12 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'archive_name_format': 'ARCHIVE_NAME',
},
storage_config={'archive_name_format': 'ARCHIVE_NAME'},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2539,12 +2500,12 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'archive_name_format': 'Documents_{hostname}-{now}', # noqa: FS003
},
storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2583,12 +2544,12 @@ def test_create_archive_with_repository_accepts_borg_placeholders():
module.create_archive(
dry_run=False,
repository_path='{fqdn}', # noqa: FS003
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['{fqdn}'], # noqa: FS003
'exclude_patterns': None,
'archive_name_format': 'Documents_{hostname}-{now}', # noqa: FS003
},
storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2626,12 +2587,12 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'extra_borg_options': {'create': '--extra --options'},
},
storage_config={'extra_borg_options': {'create': '--extra --options'}},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)
@ -2687,12 +2648,11 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
stream_processes=processes,
@ -2712,13 +2672,12 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_
module.create_archive(
dry_run=False,
repository_path='repo',
location_config={
config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
'source_directories_must_exist': True,
},
storage_config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False, used_config_paths=[]),
)

28
tests/unit/borg/test_export_tar.py
View File

@ -36,7 +36,7 @@ def test_export_tar_archive_calls_borg_with_path_parameters():
archive='archive',
paths=['path1', 'path2'],
destination_path='test.tar',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
)
@ -58,7 +58,7 @@ def test_export_tar_archive_calls_borg_with_local_path_parameters():
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
local_path='borg1',
@ -81,7 +81,7 @@ def test_export_tar_archive_calls_borg_with_remote_path_parameters():
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
config={},
local_borg_version='1.2.3',
global_arguments=flexmock(log_json=False),
remote_path='borg1',
@ -104,7 +104,7 @@ def test_export_tar_archive_calls_borg_with_umask_parameters():
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={'umask': '0770'