Move the default check state directory (#562, #638). Deprecate the "borgmatic_source_directory" option in favor of "borgmatic_runtime_directory" and "borgmatic_state_directory" (#562).
All checks were successful
build / test (push) Successful in 6m56s
build / docs (push) Successful in 2m17s

This commit is contained in:
Dan Helfman 2024-10-30 22:36:43 -07:00
parent 129f3e753c
commit 13878be254
52 changed files with 611 additions and 587 deletions

7
NEWS
View File

@ -3,6 +3,12 @@
option.
* #609: BREAKING: Apply the "working_directory" option to all actions, not just "create". This
includes repository paths, destination paths, mount points, etc.
* #562: Deprecate the "borgmatic_source_directory" option in favor of "borgmatic_runtime_directory"
and "borgmatic_state_directory".
* #562, #638: Move the default check state directory from ~/.borgmatic to
~/.local/state/borgmatic. This is more XDG-compliant and also prevents these state files from
getting backed up (unless you include them). Override this location with the new
"borgmatic_state_directory" option.
* #902: Add loading of encrypted systemd credentials. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/#using-systemd-service-credentials
* #914: Fix a confusing apparent hang when when the repository location changes, and instead
@ -32,6 +38,7 @@
* Add a "--match-archives" flag to the "prune" action.
* Add a Zabbix monitoring hook. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook
* Add a tarball of borgmatic's HTML documentation to the packages on the project page.
1.8.14
* #896: Fix an error in borgmatic rcreate/init on an empty repository directory with Borg 1.4.

View File

@ -14,7 +14,7 @@ import borgmatic.borg.extract
import borgmatic.borg.list
import borgmatic.borg.repo_list
import borgmatic.borg.state
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.config.validate
import borgmatic.execute
import borgmatic.hooks.command
@ -210,15 +210,11 @@ def make_check_time_path(config, borg_repository_id, check_type, archives_check_
"archives", etc.), and a unique hash of the archives filter flags, return a path for recording
that check's time (the time of that check last occurring).
'''
borgmatic_source_directory = os.path.expanduser(
config.get(
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
)
)
borgmatic_state_directory = borgmatic.config.paths.get_borgmatic_state_directory(config)
if check_type in ('archives', 'data'):
return os.path.join(
borgmatic_source_directory,
borgmatic_state_directory,
'checks',
borg_repository_id,
check_type,
@ -226,7 +222,7 @@ def make_check_time_path(config, borg_repository_id, check_type, archives_check_
)
return os.path.join(
borgmatic_source_directory,
borgmatic_state_directory,
'checks',
borg_repository_id,
check_type,
@ -259,7 +255,7 @@ def read_check_time(path):
def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
'''
Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
"archives", etc.), and a unique hash of the archives filter flags, return a the corresponding
"archives", etc.), and a unique hash of the archives filter flags, return the corresponding
check time or None if such a check time does not exist.
When the check type is "archives" or "data", this function probes two different paths to find
@ -297,14 +293,37 @@ def upgrade_check_times(config, borg_repository_id):
Given a configuration dict and a Borg repository ID, upgrade any corresponding check times on
disk from old-style paths to new-style paths.
Currently, the only upgrade performed is renaming an archive or data check path that looks like:
One upgrade performed is moving the checks directory from:
~/.borgmatic/checks/1234567890/archives
{borgmatic_source_directory}/checks (e.g., ~/.borgmatic/checks)
to:
~/.borgmatic/checks/1234567890/archives/all
{borgmatic_state_directory}/checks (e.g. ~/.local/state/borgmatic)
Another upgrade is renaming an archive or data check path that looks like:
{borgmatic_state_directory}/checks/1234567890/archives
to:
{borgmatic_state_directory}/checks/1234567890/archives/all
'''
borgmatic_source_checks_path = os.path.join(
borgmatic.config.paths.get_borgmatic_source_directory(config), 'checks'
)
borgmatic_state_path = borgmatic.config.paths.get_borgmatic_state_directory(config)
borgmatic_state_checks_path = os.path.join(borgmatic_state_path, 'checks')
if os.path.exists(borgmatic_source_checks_path) and not os.path.exists(
borgmatic_state_checks_path
):
logger.debug(
f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}'
)
os.makedirs(borgmatic_state_path, mode=0o700, exist_ok=True)
os.rename(borgmatic_source_checks_path, borgmatic_state_checks_path)
for check_type in ('archives', 'data'):
new_path = make_check_time_path(config, borg_repository_id, check_type, 'all')
old_path = os.path.dirname(new_path)
@ -313,7 +332,7 @@ def upgrade_check_times(config, borg_repository_id):
if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
continue
logger.debug(f'Upgrading archives check time from {old_path} to {new_path}')
logger.debug(f'Upgrading archives check time file from {old_path} to {new_path}')
try:
os.rename(old_path, temporary_path)
@ -357,7 +376,7 @@ def collect_spot_check_source_paths(
)
)
borg_environment = borgmatic.borg.environment.make_environment(config)
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
paths_output = borgmatic.execute.execute_command_and_capture_output(
create_flags + create_positional_arguments,
@ -389,13 +408,10 @@ def collect_spot_check_archive_paths(
Given a repository configuration dict, the name of the latest archive, a configuration dict, the
local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, and
the remote Borg path, collect the paths from the given archive (but only include files and
symlinks).
symlinks and exclude borgmatic runtime directories).
'''
borgmatic_source_directory = os.path.expanduser(
config.get(
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
)
)
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
borgmatic_runtime_directory = borgmatic.config.paths.get_borgmatic_runtime_directory(config)
return tuple(
path
@ -412,6 +428,7 @@ def collect_spot_check_archive_paths(
for (file_type, path) in (line.split(' ', 1),)
if file_type != BORG_DIRECTORY_FILE_TYPE
if pathlib.Path(borgmatic_source_directory) not in pathlib.Path(path).parents
if pathlib.Path(borgmatic_runtime_directory) not in pathlib.Path(path).parents
)
@ -443,7 +460,7 @@ def compare_spot_check_hashes(
int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
)
source_sample_paths = tuple(random.sample(source_paths, sample_count))
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
existing_source_sample_paths = {
source_path
for source_path in source_sample_paths

View File

@ -2,7 +2,7 @@ import logging
import shlex
import borgmatic.commands.arguments
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -68,7 +68,7 @@ def run_arbitrary_borg(
'ARCHIVE': archive if archive else '',
},
),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
from borgmatic.borg import environment, flags
from borgmatic.execute import execute_command
@ -38,7 +38,7 @@ def break_lock(
execute_command(
full_command,
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.execute
import borgmatic.logger
from borgmatic.borg import environment, flags
@ -57,7 +57,7 @@ def change_passphrase(
output_file=borgmatic.execute.DO_NOT_CAPTURE,
output_log_level=logging.ANSWER,
extra_environment=environment.make_environment(config_without_passphrase),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -2,7 +2,7 @@ import argparse
import json
import logging
import borgmatic.config.options
import borgmatic.config.paths
from borgmatic.borg import environment, feature, flags, repo_info
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -168,7 +168,7 @@ def check_archives(
+ flags.make_repository_flags(repository_path, local_borg_version)
)
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
# The Borg repair option triggers an interactive prompt, which won't work when output is
# captured. And progress messes with the terminal directly.

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
from borgmatic.borg import environment, flags
from borgmatic.execute import execute_command
@ -50,7 +50,7 @@ def compact_segments(
full_command,
output_log_level=logging.INFO,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -7,7 +7,7 @@ import stat
import tempfile
import textwrap
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, feature, flags, state
from borgmatic.execute import (
@ -357,7 +357,7 @@ def make_base_create_command(
(base Borg create command flags, Borg create command positional arguments, open pattern file
handle, open exclude file handle).
'''
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
if config.get('source_directories_must_exist', False):
check_all_source_directories_exist(
@ -527,7 +527,7 @@ def create_archive(
'''
borgmatic.logger.add_custom_log_levels()
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
borgmatic_source_directories = expand_directories(
collect_borgmatic_source_directories(config.get('borgmatic_source_directory')),
working_directory=working_directory,

View File

@ -5,7 +5,7 @@ import borgmatic.borg.environment
import borgmatic.borg.feature
import borgmatic.borg.flags
import borgmatic.borg.repo_delete
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.execute
logger = logging.getLogger(__name__)
@ -128,7 +128,7 @@ def delete_archives(
command,
output_log_level=logging.ANSWER,
extra_environment=borgmatic.borg.environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,7 +1,7 @@
import logging
import os
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -30,7 +30,7 @@ def export_key(
borgmatic.logger.add_custom_log_levels()
umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None)
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
if export_arguments.path and export_arguments.path != '-':
if os.path.exists(os.path.join(working_directory or '', export_arguments.path)):

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -71,7 +71,7 @@ def export_tar_archive(
output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
output_log_level=output_log_level,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -2,7 +2,7 @@ import logging
import os
import subprocess
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.config.validate
from borgmatic.borg import environment, feature, flags, repo_list
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -60,7 +60,7 @@ def extract_last_archive_dry_run(
execute_command(
full_extract_command,
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
@ -116,7 +116,7 @@ def extract_archive(
),
)
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
full_command = (
(local_path, 'extract')

View File

@ -1,7 +1,7 @@
import argparse
import logging
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import execute_command, execute_command_and_capture_output
@ -97,7 +97,7 @@ def display_archives_info(
remote_path,
)
borg_exit_codes = config.get('borg_exit_codes')
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
json_info = execute_command_and_capture_output(
json_command,

View File

@ -3,7 +3,7 @@ import copy
import logging
import re
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, feature, flags, repo_list
from borgmatic.execute import execute_command, execute_command_and_capture_output
@ -128,7 +128,7 @@ def capture_archive_listing(
remote_path,
),
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
@ -226,7 +226,7 @@ def list_archive(
remote_path,
),
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
@ -262,7 +262,7 @@ def list_archive(
main_command,
output_log_level=logging.ANSWER,
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -60,7 +60,7 @@ def mount_archive(
)
borg_environment = environment.make_environment(config)
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
# Don't capture the output when foreground mode is used so that ctrl-C can work properly.
if mount_arguments.foreground:

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import execute_command
@ -97,7 +97,7 @@ def prune_archives(
full_command,
output_log_level=output_log_level,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -3,7 +3,7 @@ import json
import logging
import subprocess
import borgmatic.config.options
import borgmatic.config.paths
from borgmatic.borg import environment, feature, flags, repo_info
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -97,7 +97,7 @@ def create_repository(
repo_create_command,
output_file=DO_NOT_CAPTURE,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -3,7 +3,7 @@ import logging
import borgmatic.borg.environment
import borgmatic.borg.feature
import borgmatic.borg.flags
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.execute
logger = logging.getLogger(__name__)
@ -88,7 +88,7 @@ def delete_repository(
else borgmatic.execute.DO_NOT_CAPTURE
),
extra_environment=borgmatic.borg.environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import execute_command, execute_command_and_capture_output
@ -50,7 +50,7 @@ def display_repository_info(
)
extra_environment = environment.make_environment(config)
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
borg_exit_codes = config.get('borg_exit_codes')
if repo_info_arguments.json:

View File

@ -1,7 +1,7 @@
import argparse
import logging
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import execute_command, execute_command_and_capture_output
@ -49,7 +49,7 @@ def resolve_archive_name(
output = execute_command_and_capture_output(
full_command,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
@ -158,7 +158,7 @@ def list_repository(
local_path,
remote_path,
)
working_directory = borgmatic.config.options.get_working_directory(config)
working_directory = borgmatic.config.paths.get_working_directory(config)
borg_exit_codes = config.get('borg_exit_codes')
json_listing = execute_command_and_capture_output(

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
import borgmatic.logger
from borgmatic.borg import environment, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -57,7 +57,7 @@ def transfer_archives(
output_log_level=logging.ANSWER,
output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
from borgmatic.execute import execute_command
logger = logging.getLogger(__name__)
@ -20,7 +20,7 @@ def unmount_archive(config, mount_point, local_path='borg'):
execute_command(
full_command,
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,6 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.config.paths
from borgmatic.borg import environment
from borgmatic.execute import execute_command_and_capture_output
@ -22,7 +22,7 @@ def local_borg_version(config, local_path='borg'):
output = execute_command_and_capture_output(
full_command,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -68,6 +68,17 @@ def normalize(config_filename, config):
'''
logs = normalize_sections(config_filename, config)
if config.get('borgmatic_source_directory'):
logs.append(
logging.makeLogRecord(
dict(
levelno=logging.WARNING,
levelname='WARNING',
msg=f'{config_filename}: The borgmatic_source_directory option is deprecated and will be removed from a future release. Use borgmatic_runtime_directory and borgmatic_state_directory instead.',
)
)
)
# Upgrade exclude_if_present from a string to a list.
exclude_if_present = config.get('exclude_if_present')
if isinstance(exclude_if_present, str):

View File

@ -1,11 +0,0 @@
import os
def get_working_directory(config):
'''
Given a configuration dict, get the working directory from it, first expanding any tildes.
'''
try:
return os.path.expanduser(config.get('working_directory', '')) or None
except TypeError:
return None

65
borgmatic/config/paths.py Normal file
View File

@ -0,0 +1,65 @@
import os
def expand_user_in_path(path):
'''
Given a directory path, expand any tildes in it.
'''
try:
return os.path.expanduser(path or '') or None
except TypeError:
return None
def get_working_directory(config): # pragma: no cover
'''
Given a configuration dict, get the working directory from it, expanding any tildes.
'''
return expand_user_in_path(config.get('working_directory'))
def get_borgmatic_source_directory(config):
'''
Given a configuration dict, get the (deprecated) borgmatic source directory, expanding any
tildes. Defaults to ~/.borgmatic.
'''
return expand_user_in_path(config.get('borgmatic_source_directory') or '~/.borgmatic')
def get_borgmatic_runtime_directory(config):
'''
Given a configuration dict, get the borgmatic runtime directory used for storing temporary
runtime data like streaming database dumps and bootstrap metadata. Defaults to the
"borgmatic_source_directory" value (deprecated) or $XDG_RUNTIME_DIR/borgmatic or
/var/run/$UID/borgmatic.
'''
return expand_user_in_path(
config.get('borgmatic_runtime_directory')
or config.get('borgmatic_source_directory')
or os.path.join(
os.environ.get(
'XDG_RUNTIME_DIR',
f'/var/run/{os.getuid()}',
),
'borgmatic',
)
)
def get_borgmatic_state_directory(config):
'''
Given a configuration dict, get the borgmatic state directory used for storing borgmatic state
files like records of when checks last ran. Defaults to the "borgmatic_source_directory" value
(deprecated) or $XDG_STATE_HOME/borgmatic or ~/.local/state/borgmatic.
'''
return expand_user_in_path(
config.get('borgmatic_state_directory')
or config.get('borgmatic_source_directory')
or os.path.join(
os.environ.get(
'XDG_STATE_HOME',
'~/.local/state',
),
'borgmatic',
)
)

View File

@ -207,11 +207,25 @@ properties:
borgmatic_source_directory:
type: string
description: |
Path for additional source files used for temporary internal state
like borgmatic database dumps. Note that changing this path prevents
"borgmatic restore" from finding any database dumps created before
the change. Defaults to ~/.borgmatic
Deprecated. Replaced by borgmatic_runtime_directory and
borgmatic_state_directory. Defaults to ~/.borgmatic
example: /tmp/borgmatic
borgmatic_runtime_directory:
type: string
description: |
Path for storing temporary runtime data like streaming database
dumps and bootstrap metadata. Defaults to the
borgmatic_source_directory value (deprecated) or
$XDG_RUNTIME_DIR/borgmatic or /var/run/$UID/borgmatic.
example: /run/user/1001/borgmatic
borgmatic_state_directory:
type: string
description: |
Path for storing borgmatic state files like records of when checks
last ran. Defaults to the borgmatic_source_directory value
(deprecated) or $XDG_STATE_HOME/borgmatic or
~/.local/state/borgmatic.
example: /var/lib/borgmatic
store_config_files:
type: boolean
description: |

View File

@ -268,54 +268,45 @@ def test_make_archives_check_id_with_empty_flags_returns_none():
def test_make_check_time_path_with_borgmatic_source_directory_includes_it():
flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return(
'/home/user/.borgmatic'
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_state_directory'
).and_return(
'/home/user/.local/state/borgmatic',
)
assert (
module.make_check_time_path(
{'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'archives', '5678'
)
== '/home/user/.borgmatic/checks/1234/archives/5678'
)
def test_make_check_time_path_without_borgmatic_source_directory_uses_default():
flexmock(module.os.path).should_receive('expanduser').with_args(
module.borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
).and_return('/home/user/.borgmatic')
assert (
module.make_check_time_path({}, '1234', 'archives', '5678')
== '/home/user/.borgmatic/checks/1234/archives/5678'
== '/home/user/.local/state/borgmatic/checks/1234/archives/5678'
)
def test_make_check_time_path_with_archives_check_and_no_archives_check_id_defaults_to_all():
flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return(
'/home/user/.borgmatic'
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_state_directory'
).and_return(
'/home/user/.local/state/borgmatic',
)
assert (
module.make_check_time_path(
{'borgmatic_source_directory': '~/.borgmatic'},
{},
'1234',
'archives',
)
== '/home/user/.borgmatic/checks/1234/archives/all'
== '/home/user/.local/state/borgmatic/checks/1234/archives/all'
)
def test_make_check_time_path_with_repositories_check_ignores_archives_check_id():
flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return(
'/home/user/.borgmatic'
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_state_directory'
).and_return(
'/home/user/.local/state/borgmatic',
)
assert (
module.make_check_time_path(
{'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'repository', '5678'
)
== '/home/user/.borgmatic/checks/1234/repository'
module.make_check_time_path({}, '1234', 'repository', '5678')
== '/home/user/.local/state/borgmatic/checks/1234/repository'
)
@ -376,8 +367,69 @@ def test_probe_for_check_time_returns_none_when_no_check_time_found():
assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) is None
def test_upgrade_check_times_moves_checks_from_borgmatic_source_directory_to_state_directory():
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_source_directory'
).and_return('/home/user/.borgmatic')
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_state_directory'
).and_return('/home/user/.local/state/borgmatic')
flexmock(module.os.path).should_receive('exists').with_args(
'/home/user/.borgmatic/checks'
).and_return(True)
flexmock(module.os.path).should_receive('exists').with_args(
'/home/user/.local/state/borgmatic/checks'
).and_return(False)
flexmock(module.os).should_receive('makedirs')
flexmock(module.os).should_receive('rename').with_args(
'/home/user/.borgmatic/checks', '/home/user/.local/state/borgmatic/checks'
).once()
flexmock(module).should_receive('make_check_time_path').and_return(
'/home/user/.local/state/borgmatic/checks/1234/archives/all'
)
flexmock(module.os.path).should_receive('isfile').and_return(False)
flexmock(module.os).should_receive('mkdir').never()
module.upgrade_check_times(flexmock(), flexmock())
def test_upgrade_check_times_with_checks_already_in_borgmatic_state_directory_does_not_move_anything():
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_source_directory'
).and_return('/home/user/.borgmatic')
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_state_directory'
).and_return('/home/user/.local/state/borgmatic')
flexmock(module.os.path).should_receive('exists').with_args(
'/home/user/.borgmatic/checks'
).and_return(True)
flexmock(module.os.path).should_receive('exists').with_args(
'/home/user/.local/state/borgmatic/checks'
).and_return(True)
flexmock(module.os).should_receive('makedirs').never()
flexmock(module.os).should_receive('rename').never()
flexmock(module).should_receive('make_check_time_path').and_return(
'/home/user/.local/state/borgmatic/checks/1234/archives/all'
)
flexmock(module.os.path).should_receive('isfile').and_return(False)
flexmock(module.os).should_receive('rename').never()
flexmock(module.os).should_receive('mkdir').never()
module.upgrade_check_times(flexmock(), flexmock())
def test_upgrade_check_times_renames_old_check_paths_to_all():
base_path = '~/.borgmatic/checks/1234'
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_source_directory'
).and_return('/home/user/.borgmatic')
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_state_directory'
).and_return('/home/user/.local/state/borgmatic')
flexmock(module.os.path).should_receive('exists').and_return(False)
base_path = '/home/user/.local/state/borgmatic/checks/1234'
flexmock(module).should_receive('make_check_time_path').with_args(
object, object, 'archives', 'all'
).and_return(f'{base_path}/archives/all')
@ -408,7 +460,15 @@ def test_upgrade_check_times_renames_old_check_paths_to_all():
def test_upgrade_check_times_renames_data_check_paths_when_archives_paths_are_already_upgraded():
base_path = '~/.borgmatic/checks/1234'
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_source_directory'
).and_return('/home/user/.borgmatic')
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_state_directory'
).and_return('/home/user/.local/state/borgmatic')
flexmock(module.os.path).should_receive('exists').and_return(False)
base_path = '/home/user/.local/state/borgmatic/checks/1234'
flexmock(module).should_receive('make_check_time_path').with_args(
object, object, 'archives', 'all'
).and_return(f'{base_path}/archives/all')
@ -435,9 +495,17 @@ def test_upgrade_check_times_renames_data_check_paths_when_archives_paths_are_al
module.upgrade_check_times(flexmock(), flexmock())
def test_upgrade_check_times_skips_missing_check_paths():
def test_upgrade_check_times_skips_already_upgraded_check_paths():
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_source_directory'
).and_return('/home/user/.borgmatic')
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_state_directory'
).and_return('/home/user/.local/state/borgmatic')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module).should_receive('make_check_time_path').and_return(
'~/.borgmatic/checks/1234/archives/all'
'/home/user/.local/state/borgmatic/checks/1234/archives/all'
)
flexmock(module.os.path).should_receive('isfile').and_return(False)
flexmock(module.os).should_receive('rename').never()
@ -447,7 +515,15 @@ def test_upgrade_check_times_skips_missing_check_paths():
def test_upgrade_check_times_renames_stale_temporary_check_path():
base_path = '~/.borgmatic/checks/1234'
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_source_directory'
).and_return('/home/user/.borgmatic')
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_state_directory'
).and_return('/home/user/.local/state/borgmatic')
flexmock(module.os.path).should_receive('exists').and_return(False)
base_path = '/home/borgmatic/.local/state/checks/1234'
flexmock(module).should_receive('make_check_time_path').with_args(
object, object, 'archives', 'all'
).and_return(f'{base_path}/archives/all')
@ -497,9 +573,7 @@ def test_collect_spot_check_source_paths_parses_borg_output():
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
@ -537,9 +611,7 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
@ -577,9 +649,7 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
@ -617,9 +687,7 @@ def test_collect_spot_check_source_paths_skips_directories():
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
@ -642,6 +710,12 @@ def test_collect_spot_check_source_paths_skips_directories():
def test_collect_spot_check_archive_paths_excludes_directories():
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_source_directory'
).and_return('/home/user/.borgmatic')
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_runtime_directory'
).and_return('/var/run/1001/borgmatic')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
(
'f /etc/path',
@ -662,6 +736,12 @@ def test_collect_spot_check_archive_paths_excludes_directories():
def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_directory():
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_source_directory'
).and_return('/root/.borgmatic')
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_runtime_directory'
).and_return('/var/run/0/borgmatic')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
(
'f /etc/path',
@ -680,6 +760,31 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_dire
) == ('/etc/path',)
def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory():
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_source_directory'
).and_return('/root.borgmatic')
flexmock(module.borgmatic.config.paths).should_receive(
'get_borgmatic_runtime_directory'
).and_return('/var/run/0/borgmatic')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
(
'f /etc/path',
'f /var/run/0/borgmatic/some/thing',
)
)
assert module.collect_spot_check_archive_paths(
repository={'path': 'repo'},
archive='archive',
config={'borgmatic_runtime_directory': '/var/run/0/borgmatic'},
local_borg_version=flexmock(),
global_arguments=flexmock(),
local_path=flexmock(),
remote_path=flexmock(),
) == ('/etc/path',)
def test_collect_spot_check_source_paths_uses_working_directory():
flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
{'hook1': False, 'hook2': True}
@ -700,7 +805,7 @@ def test_collect_spot_check_source_paths_uses_working_directory():
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
'/working/dir'
)
flexmock(module.borgmatic.execute).should_receive(
@ -725,7 +830,7 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
@ -766,7 +871,7 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
@ -807,7 +912,7 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
@ -845,7 +950,7 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
@ -882,7 +987,7 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching()
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').with_args('/foo').and_return(True)
@ -919,7 +1024,7 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
@ -965,7 +1070,7 @@ def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
'/working/dir',
)
flexmock(module.os.path).should_receive('exists').with_args('/working/dir/foo').and_return(True)