Deprecate the "borgmatic_source_directory" option in favor of "user_runtime_directory" and "user_state_directory" (#562). Move the default borgmatic streaming database dump and bootstrap metadata location on disk (#562). With Borg 1.4+, store database dumps and bootstrap metadata in a "/borgmatic" directory within a backup archive (#838). Add "--local-path", "--remote-path", and "--user-runtime-directory" flags to the "config bootstrap" action.
This commit is contained in:
parent
13878be254
commit
814cdb4b87
20
NEWS
20
NEWS
@ -3,12 +3,20 @@
|
||||
option.
|
||||
* #609: BREAKING: Apply the "working_directory" option to all actions, not just "create". This
|
||||
includes repository paths, destination paths, mount points, etc.
|
||||
* #562: Deprecate the "borgmatic_source_directory" option in favor of "borgmatic_runtime_directory"
|
||||
and "borgmatic_state_directory".
|
||||
* #562: Deprecate the "borgmatic_source_directory" option in favor of "user_runtime_directory"
|
||||
and "user_state_directory".
|
||||
* #562: BREAKING: Move the default borgmatic streaming database dump and bootstrap metadata
|
||||
directory from ~/.borgmatic to /run/user/$UID/borgmatic, which is more XDG-compliant. Override
|
||||
this location with the new "user_runtime_directory" option. Existing archives with database dumps
|
||||
at the old location are still restorable.
|
||||
* #562, #638: Move the default check state directory from ~/.borgmatic to
|
||||
~/.local/state/borgmatic. This is more XDG-compliant and also prevents these state files from
|
||||
getting backed up (unless you include them). Override this location with the new
|
||||
"borgmatic_state_directory" option.
|
||||
getting backed up (unless you explicitly include them). Override this location with the new
|
||||
"user_state_directory" option. After the first time you run the "check" action with borgmatic
|
||||
1.9.0, you can safely delete the ~/.borgmatic directory.
|
||||
* #838: With Borg 1.4+, store database dumps and bootstrap metadata in a "/borgmatic" directory
|
||||
within a backup archive, so the path doesn't depend on the current user. This means that you can
|
||||
now backup as one user and restore or bootstrap as another user, among other use cases.
|
||||
* #902: Add loading of encrypted systemd credentials. See the documentation for more information:
|
||||
https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/#using-systemd-service-credentials
|
||||
* #914: Fix a confusing apparent hang when when the repository location changes, and instead
|
||||
@ -36,6 +44,10 @@
|
||||
* Update the "--match-archives" and "--archive" flags to support Borg 2 series names or archive
|
||||
hashes.
|
||||
* Add a "--match-archives" flag to the "prune" action.
|
||||
* Add "--local-path" and "--remote-path" flags to the "config bootstrap" action for setting the
|
||||
Borg executable paths used for bootstrapping.
|
||||
* Add a "--user-runtime-directory" flag to the "config bootstrap" action for helping borgmatic
|
||||
locate the bootstrap metadata stored in an archive.
|
||||
* Add a Zabbix monitoring hook. See the documentation for more information:
|
||||
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook
|
||||
* Add a tarball of borgmatic's HTML documentation to the packages on the project page.
|
||||
|
||||
@ -368,7 +368,7 @@ def collect_spot_check_source_paths(
|
||||
config_paths=(),
|
||||
local_borg_version=local_borg_version,
|
||||
global_arguments=global_arguments,
|
||||
borgmatic_source_directories=(),
|
||||
borgmatic_runtime_directories=(),
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
list_files=True,
|
||||
@ -427,6 +427,7 @@ def collect_spot_check_archive_paths(
|
||||
)
|
||||
for (file_type, path) in (line.split(' ', 1),)
|
||||
if file_type != BORG_DIRECTORY_FILE_TYPE
|
||||
if pathlib.Path('/borgmatic') not in pathlib.Path(path).parents
|
||||
if pathlib.Path(borgmatic_source_directory) not in pathlib.Path(path).parents
|
||||
if pathlib.Path(borgmatic_runtime_directory) not in pathlib.Path(path).parents
|
||||
)
|
||||
|
||||
@ -4,51 +4,68 @@ import os
|
||||
|
||||
import borgmatic.borg.extract
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.hooks.command
|
||||
from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
|
||||
def make_bootstrap_config(bootstrap_arguments):
|
||||
'''
|
||||
Given the bootstrap arguments as an argparse.Namespace (containing the repository and archive
|
||||
name, borgmatic source directory, destination directory, and whether to strip components), the
|
||||
global arguments as an argparse.Namespace (containing the dry run flag and the local borg
|
||||
version), return the config paths from the manifest.json file in the borgmatic source directory
|
||||
after extracting it from the repository.
|
||||
Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict.
|
||||
'''
|
||||
return {
|
||||
'ssh_command': bootstrap_arguments.ssh_command,
|
||||
# In case the repo has been moved or is accessed from a different path at the point of
|
||||
# bootstrapping.
|
||||
'relocated_repo_access_is_ok': True,
|
||||
}
|
||||
|
||||
|
||||
def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version):
|
||||
'''
|
||||
Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the
|
||||
repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory,
|
||||
borgmatic source directory, destination directory, and whether to strip components), the global
|
||||
arguments as an argparse.Namespace (containing the dry run flag and the local borg version),
|
||||
return the config paths from the manifest.json file in the borgmatic source directory or runtime
|
||||
directory after extracting it from the repository archive.
|
||||
|
||||
Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
|
||||
expected configuration path data.
|
||||
'''
|
||||
borgmatic_source_directory = (
|
||||
bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(
|
||||
{'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory}
|
||||
)
|
||||
borgmatic_manifest_path = os.path.expanduser(
|
||||
os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
|
||||
borgmatic_runtime_directory = borgmatic.config.paths.get_borgmatic_runtime_directory(
|
||||
{'user_runtime_directory': bootstrap_arguments.user_runtime_directory}
|
||||
)
|
||||
config = {'ssh_command': bootstrap_arguments.ssh_command}
|
||||
config = make_bootstrap_config(bootstrap_arguments)
|
||||
|
||||
extract_process = borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
bootstrap_arguments.repository,
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
# Probe for the manifest file in multiple locations, as the default location has moved to the
|
||||
# borgmatic runtime directory (which get stored as just "/borgmatic" with Borg 1.4+). But we
|
||||
# still want to support reading the manifest from previously created archives as well.
|
||||
for base_directory in ('borgmatic', borgmatic_runtime_directory, borgmatic_source_directory):
|
||||
borgmatic_manifest_path = os.path.join(base_directory, 'bootstrap', 'manifest.json')
|
||||
|
||||
extract_process = borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
bootstrap_arguments.repository,
|
||||
bootstrap_arguments.archive,
|
||||
archive_name,
|
||||
[borgmatic_manifest_path],
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
),
|
||||
[borgmatic_manifest_path],
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
extract_to_stdout=True,
|
||||
)
|
||||
manifest_json = extract_process.stdout.read()
|
||||
local_path=bootstrap_arguments.local_path,
|
||||
remote_path=bootstrap_arguments.remote_path,
|
||||
extract_to_stdout=True,
|
||||
)
|
||||
manifest_json = extract_process.stdout.read()
|
||||
|
||||
if not manifest_json:
|
||||
if manifest_json:
|
||||
break
|
||||
else:
|
||||
raise ValueError(
|
||||
'Cannot read configuration paths from archive due to missing bootstrap manifest'
|
||||
)
|
||||
@ -75,27 +92,32 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
|
||||
Raise ValueError if the bootstrap configuration could not be loaded.
|
||||
Raise CalledProcessError or OSError if Borg could not be run.
|
||||
'''
|
||||
manifest_config_paths = get_config_paths(
|
||||
bootstrap_arguments, global_arguments, local_borg_version
|
||||
config = make_bootstrap_config(bootstrap_arguments)
|
||||
archive_name = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
bootstrap_arguments.repository,
|
||||
bootstrap_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=bootstrap_arguments.local_path,
|
||||
remote_path=bootstrap_arguments.remote_path,
|
||||
)
|
||||
manifest_config_paths = get_config_paths(
|
||||
archive_name, bootstrap_arguments, global_arguments, local_borg_version
|
||||
)
|
||||
config = {'ssh_command': bootstrap_arguments.ssh_command}
|
||||
|
||||
logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
|
||||
|
||||
borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
bootstrap_arguments.repository,
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
bootstrap_arguments.repository,
|
||||
bootstrap_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
),
|
||||
archive_name,
|
||||
[config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=bootstrap_arguments.local_path,
|
||||
remote_path=bootstrap_arguments.remote_path,
|
||||
extract_to_stdout=False,
|
||||
destination_path=bootstrap_arguments.destination,
|
||||
strip_components=bootstrap_arguments.strip_components,
|
||||
|
||||
@ -5,7 +5,7 @@ import os
|
||||
|
||||
import borgmatic.actions.json
|
||||
import borgmatic.borg.create
|
||||
import borgmatic.borg.state
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.hooks.command
|
||||
import borgmatic.hooks.dispatch
|
||||
@ -22,12 +22,9 @@ def create_borgmatic_manifest(config, config_paths, dry_run):
|
||||
if dry_run:
|
||||
return
|
||||
|
||||
borgmatic_source_directory = config.get(
|
||||
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||
)
|
||||
|
||||
borgmatic_manifest_path = os.path.expanduser(
|
||||
os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
|
||||
borgmatic_runtime_directory = borgmatic.config.paths.get_borgmatic_runtime_directory(config)
|
||||
borgmatic_manifest_path = os.path.join(
|
||||
borgmatic_runtime_directory, 'bootstrap', 'manifest.json'
|
||||
)
|
||||
|
||||
if not os.path.exists(borgmatic_manifest_path):
|
||||
|
||||
@ -1,12 +1,15 @@
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
import borgmatic.borg.extract
|
||||
import borgmatic.borg.list
|
||||
import borgmatic.borg.mount
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.borg.state
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.hooks.dispatch
|
||||
import borgmatic.hooks.dump
|
||||
@ -61,6 +64,37 @@ def get_configured_data_source(
|
||||
)
|
||||
|
||||
|
||||
def strip_path_prefix_from_extracted_dump_destination(
|
||||
destination_path, borgmatic_runtime_directory
|
||||
):
|
||||
'''
|
||||
Directory-format dump files get extracted into a temporary directory containing a path prefix
|
||||
that depends how the files were stored in the archive. So, given the destination path where the
|
||||
dump was extracted and the borgmatic runtime directory, move the dump files such that the
|
||||
restore doesn't have to deal with that varying path prefix.
|
||||
|
||||
For instance, if the dump was extracted to:
|
||||
|
||||
/run/user/0/borgmatic/tmp1234/borgmatic/postgresql_databases/test/...
|
||||
|
||||
or:
|
||||
|
||||
/run/user/0/borgmatic/tmp1234/root/.borgmatic/postgresql_databases/test/...
|
||||
|
||||
then this function moves it to:
|
||||
|
||||
/run/user/0/borgmatic/postgresql_databases/test/...
|
||||
'''
|
||||
for subdirectory_path, _, _ in os.walk(destination_path):
|
||||
databases_directory = os.path.basename(subdirectory_path)
|
||||
|
||||
if not databases_directory.endswith('_databases'):
|
||||
continue
|
||||
|
||||
os.rename(subdirectory_path, os.path.join(borgmatic_runtime_directory, databases_directory))
|
||||
break
|
||||
|
||||
|
||||
def restore_single_data_source(
|
||||
repository,
|
||||
config,
|
||||
@ -72,7 +106,7 @@ def restore_single_data_source(
|
||||
hook_name,
|
||||
data_source,
|
||||
connection_params,
|
||||
): # pragma: no cover
|
||||
):
|
||||
'''
|
||||
Given (among other things) an archive name, a data source hook name, the hostname, port,
|
||||
username/password as connection params, and a configured data source configuration dict, restore
|
||||
@ -82,31 +116,48 @@ def restore_single_data_source(
|
||||
f'{repository.get("label", repository["path"])}: Restoring data source {data_source["name"]}'
|
||||
)
|
||||
|
||||
dump_pattern = borgmatic.hooks.dispatch.call_hooks(
|
||||
'make_data_source_dump_pattern',
|
||||
dump_patterns = borgmatic.hooks.dispatch.call_hooks(
|
||||
'make_data_source_dump_patterns',
|
||||
config,
|
||||
repository['path'],
|
||||
borgmatic.hooks.dump.DATA_SOURCE_HOOK_NAMES,
|
||||
data_source['name'],
|
||||
)[hook_name]
|
||||
borgmatic_runtime_directory = borgmatic.config.paths.get_borgmatic_runtime_directory(config)
|
||||
|
||||
# Kick off a single data source extract to stdout.
|
||||
extract_process = borgmatic.borg.extract.extract_archive(
|
||||
dry_run=global_arguments.dry_run,
|
||||
repository=repository['path'],
|
||||
archive=archive_name,
|
||||
paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
|
||||
config=config,
|
||||
local_borg_version=local_borg_version,
|
||||
global_arguments=global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
destination_path='/',
|
||||
# A directory format dump isn't a single file, and therefore can't extract
|
||||
# to stdout. In this case, the extract_process return value is None.
|
||||
extract_to_stdout=bool(data_source.get('format') != 'directory'),
|
||||
destination_path = (
|
||||
tempfile.mkdtemp(dir=borgmatic_runtime_directory)
|
||||
if data_source.get('format') == 'directory'
|
||||
else None
|
||||
)
|
||||
|
||||
try:
|
||||
# Kick off a single data source extract. If using a directory format, extract to a temporary
|
||||
# directory. Otheriwes extract the single dump file to stdout.
|
||||
extract_process = borgmatic.borg.extract.extract_archive(
|
||||
dry_run=global_arguments.dry_run,
|
||||
repository=repository['path'],
|
||||
archive=archive_name,
|
||||
paths=[borgmatic.hooks.dump.convert_glob_patterns_to_borg_pattern(dump_patterns)],
|
||||
config=config,
|
||||
local_borg_version=local_borg_version,
|
||||
global_arguments=global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
destination_path=destination_path,
|
||||
# A directory format dump isn't a single file, and therefore can't extract
|
||||
# to stdout. In this case, the extract_process return value is None.
|
||||
extract_to_stdout=bool(data_source.get('format') != 'directory'),
|
||||
)
|
||||
|
||||
if destination_path and not global_arguments.dry_run:
|
||||
strip_path_prefix_from_extracted_dump_destination(
|
||||
destination_path, borgmatic_runtime_directory
|
||||
)
|
||||
finally:
|
||||
if destination_path and not global_arguments.dry_run:
|
||||
shutil.rmtree(destination_path, ignore_errors=True)
|
||||
|
||||
# Run a single data source restore, consuming the extract stdout (if any).
|
||||
borgmatic.hooks.dispatch.call_hooks(
|
||||
function_name='restore_data_source_dump',
|
||||
@ -135,11 +186,14 @@ def collect_archive_data_source_names(
|
||||
query the archive for the names of data sources it contains as dumps and return them as a dict
|
||||
from hook name to a sequence of data source names.
|
||||
'''
|
||||
borgmatic_source_directory = os.path.expanduser(
|
||||
config.get(
|
||||
'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||
)
|
||||
).lstrip('/')
|
||||
borgmatic_source_directory = str(
|
||||
pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
|
||||
)
|
||||
borgmatic_runtime_directory = borgmatic.config.paths.get_borgmatic_runtime_directory(config)
|
||||
|
||||
# Probe for the data source dumps in multiple locations, as the default location has moved to
|
||||
# the borgmatic runtime directory (which get stored as just "/borgmatic" with Borg 1.4+). But we
|
||||
# still want to support reading dumps from previously created archives as well.
|
||||
dump_paths = borgmatic.borg.list.capture_archive_listing(
|
||||
repository,
|
||||
archive,
|
||||
@ -148,10 +202,12 @@ def collect_archive_data_source_names(
|
||||
global_arguments,
|
||||
list_paths=[
|
||||
'sh:'
|
||||
+ os.path.expanduser(
|
||||
borgmatic.hooks.dump.make_data_source_dump_path(borgmatic_source_directory, pattern)
|
||||
+ borgmatic.hooks.dump.make_data_source_dump_path(base_directory, '*_databases/*/*')
|
||||
for base_directory in (
|
||||
'borgmatic',
|
||||
borgmatic_runtime_directory.lstrip('/'),
|
||||
borgmatic_source_directory.lstrip('/'),
|
||||
)
|
||||
for pattern in ('*_databases/*/*',)
|
||||
],
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
@ -162,17 +218,28 @@ def collect_archive_data_source_names(
|
||||
archive_data_source_names = {}
|
||||
|
||||
for dump_path in dump_paths:
|
||||
try:
|
||||
(hook_name, _, data_source_name) = dump_path.split(
|
||||
borgmatic_source_directory + os.path.sep, 1
|
||||
)[1].split(os.path.sep)[0:3]
|
||||
except (ValueError, IndexError):
|
||||
if not dump_path:
|
||||
continue
|
||||
|
||||
for base_directory in (
|
||||
'borgmatic',
|
||||
borgmatic_runtime_directory,
|
||||
borgmatic_source_directory,
|
||||
):
|
||||
try:
|
||||
(hook_name, _, data_source_name) = dump_path.split(base_directory + os.path.sep, 1)[
|
||||
1
|
||||
].split(os.path.sep)[0:3]
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
else:
|
||||
if data_source_name not in archive_data_source_names.get(hook_name, []):
|
||||
archive_data_source_names.setdefault(hook_name, []).extend([data_source_name])
|
||||
break
|
||||
else:
|
||||
logger.warning(
|
||||
f'{repository}: Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
|
||||
)
|
||||
else:
|
||||
if data_source_name not in archive_data_source_names.get(hook_name, []):
|
||||
archive_data_source_names.setdefault(hook_name, []).extend([data_source_name])
|
||||
|
||||
return archive_data_source_names
|
||||
|
||||
@ -243,7 +310,7 @@ def ensure_data_sources_found(restore_names, remaining_restore_names, found_name
|
||||
)
|
||||
|
||||
if not combined_restore_names and not found_names:
|
||||
raise ValueError('No data sources were found to restore')
|
||||
raise ValueError('No data source dumps were found to restore')
|
||||
|
||||
missing_names = sorted(set(combined_restore_names) - set(found_names))
|
||||
if missing_names:
|
||||
|
||||
@ -9,7 +9,7 @@ import textwrap
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.logger
|
||||
from borgmatic.borg import environment, feature, flags, state
|
||||
from borgmatic.borg import environment, feature, flags
|
||||
from borgmatic.execute import (
|
||||
DO_NOT_CAPTURE,
|
||||
execute_command,
|
||||
@ -221,18 +221,13 @@ def make_list_filter_flags(local_borg_version, dry_run):
|
||||
return f'{base_flags}-'
|
||||
|
||||
|
||||
def collect_borgmatic_source_directories(borgmatic_source_directory):
|
||||
def collect_borgmatic_runtime_directories(borgmatic_runtime_directory):
|
||||
'''
|
||||
Return a list of borgmatic-specific source directories used for state like database backups.
|
||||
Return a list of borgmatic-specific runtime directories used for temporary runtime data like
|
||||
streaming database dumps and bootstrap metadata. If no such directories exist, return an empty
|
||||
list.
|
||||
'''
|
||||
if not borgmatic_source_directory:
|
||||
borgmatic_source_directory = state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||
|
||||
return (
|
||||
[borgmatic_source_directory]
|
||||
if os.path.exists(os.path.expanduser(borgmatic_source_directory))
|
||||
else []
|
||||
)
|
||||
return [borgmatic_runtime_directory] if os.path.exists(borgmatic_runtime_directory) else []
|
||||
|
||||
|
||||
ROOT_PATTERN_PREFIX = 'R '
|
||||
@ -342,7 +337,7 @@ def make_base_create_command(
|
||||
config_paths,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
borgmatic_source_directories,
|
||||
borgmatic_runtime_directories,
|
||||
local_path='borg',
|
||||
remote_path=None,
|
||||
progress=False,
|
||||
@ -368,7 +363,7 @@ def make_base_create_command(
|
||||
map_directories_to_devices(
|
||||
expand_directories(
|
||||
tuple(config.get('source_directories', ()))
|
||||
+ borgmatic_source_directories
|
||||
+ borgmatic_runtime_directories
|
||||
+ tuple(config_paths if config.get('store_config_files', True) else ()),
|
||||
working_directory=working_directory,
|
||||
)
|
||||
@ -479,7 +474,7 @@ def make_base_create_command(
|
||||
local_path,
|
||||
working_directory,
|
||||
borg_environment,
|
||||
skip_directories=borgmatic_source_directories,
|
||||
skip_directories=borgmatic_runtime_directories,
|
||||
)
|
||||
|
||||
if special_file_paths:
|
||||
@ -528,8 +523,10 @@ def create_archive(
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
borgmatic_source_directories = expand_directories(
|
||||
collect_borgmatic_source_directories(config.get('borgmatic_source_directory')),
|
||||
borgmatic_runtime_directories = expand_directories(
|
||||
collect_borgmatic_runtime_directories(
|
||||
borgmatic.config.paths.get_borgmatic_runtime_directory(config)
|
||||
),
|
||||
working_directory=working_directory,
|
||||
)
|
||||
|
||||
@ -541,7 +538,7 @@ def create_archive(
|
||||
config_paths,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
borgmatic_source_directories,
|
||||
borgmatic_runtime_directories,
|
||||
local_path,
|
||||
remote_path,
|
||||
progress,
|
||||
|
||||
@ -132,10 +132,9 @@ def extract_archive(
|
||||
+ (('--progress',) if progress else ())
|
||||
+ (('--stdout',) if extract_to_stdout else ())
|
||||
+ flags.make_repository_archive_flags(
|
||||
# Make the repository path absolute so the destination directory
|
||||
# used below via changing the working directory doesn't prevent
|
||||
# Borg from finding the repo. But also apply the user's configured
|
||||
# working directory (if any) to the repo path.
|
||||
# Make the repository path absolute so the destination directory used below via changing
|
||||
# the working directory doesn't prevent Borg from finding the repo. But also apply the
|
||||
# user's configured working directory (if any) to the repo path.
|
||||
borgmatic.config.validate.normalize_repository_path(
|
||||
os.path.join(working_directory or '', repository)
|
||||
),
|
||||
|
||||
@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def local_borg_version(config, local_path='borg'):
|
||||
'''
|
||||
Given a configuration dict and a local Borg binary path, return a version string for it.
|
||||
Given a configuration dict and a local Borg executable path, return a version string for it.
|
||||
|
||||
Raise OSError or CalledProcessError if there is a problem running Borg.
|
||||
Raise ValueError if the version cannot be parsed.
|
||||
|
||||
@ -74,11 +74,11 @@ def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments
|
||||
for action_name, parsed in parsed_arguments.items():
|
||||
for value in vars(parsed).values():
|
||||
if isinstance(value, str):
|
||||
if value in ACTION_ALIASES.keys():
|
||||
if value in ACTION_ALIASES.keys() and value in remaining_arguments:
|
||||
remaining_arguments.remove(value)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
if item in ACTION_ALIASES.keys():
|
||||
if item in ACTION_ALIASES.keys() and item in remaining_arguments:
|
||||
remaining_arguments.remove(item)
|
||||
|
||||
return tuple(remaining_arguments)
|
||||
@ -864,9 +864,23 @@ def make_parsers():
|
||||
help='Path of repository to extract config files from, quoted globs supported',
|
||||
required=True,
|
||||
)
|
||||
config_bootstrap_group.add_argument(
|
||||
'--local-path',
|
||||
help='Alternate Borg local executable. Defaults to "borg"',
|
||||
default='borg',
|
||||
)
|
||||
config_bootstrap_group.add_argument(
|
||||
'--remote-path',
|
||||
help='Alternate Borg remote executable. Defaults to "borg"',
|
||||
default='borg',
|
||||
)
|
||||
config_bootstrap_group.add_argument(
|
||||
'--user-runtime-directory',
|
||||
help='Path used for temporary runtime data like bootstrap metadata. Defaults to $XDG_RUNTIME_DIR or /var/run/$UID',
|
||||
)
|
||||
config_bootstrap_group.add_argument(
|
||||
'--borgmatic-source-directory',
|
||||
help='Path that stores the config files used to create an archive and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic',
|
||||
help='Deprecated. Path formerly used for temporary runtime data like bootstrap metadata. Defaults to ~/.borgmatic',
|
||||
)
|
||||
config_bootstrap_group.add_argument(
|
||||
'--archive',
|
||||
|
||||
@ -694,7 +694,9 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
|
||||
if 'bootstrap' in arguments:
|
||||
try:
|
||||
# No configuration file is needed for bootstrap.
|
||||
local_borg_version = borg_version.local_borg_version({}, 'borg')
|
||||
local_borg_version = borg_version.local_borg_version(
|
||||
{}, arguments['bootstrap'].local_path
|
||||
)
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
yield from log_error_records('Error getting local Borg version', error)
|
||||
return
|
||||
|
||||
@ -29,18 +29,21 @@ def get_borgmatic_source_directory(config):
|
||||
def get_borgmatic_runtime_directory(config):
|
||||
'''
|
||||
Given a configuration dict, get the borgmatic runtime directory used for storing temporary
|
||||
runtime data like streaming database dumps and bootstrap metadata. Defaults to the
|
||||
"borgmatic_source_directory" value (deprecated) or $XDG_RUNTIME_DIR/borgmatic or
|
||||
/var/run/$UID/borgmatic.
|
||||
runtime data like streaming database dumps and bootstrap metadata. Defaults to
|
||||
$XDG_RUNTIME_DIR/./borgmatic or /run/user/$UID/./borgmatic.
|
||||
|
||||
The "/./" is taking advantage of a Borg feature such that the part of the path before the "/./"
|
||||
does not get stored in the file path within an archive. That way, the path of the runtime
|
||||
directory can change without leaving database dumps within an archive inaccessible.
|
||||
'''
|
||||
return expand_user_in_path(
|
||||
config.get('borgmatic_runtime_directory')
|
||||
or config.get('borgmatic_source_directory')
|
||||
or os.path.join(
|
||||
os.environ.get(
|
||||
os.path.join(
|
||||
config.get('user_runtime_directory')
|
||||
or os.environ.get(
|
||||
'XDG_RUNTIME_DIR',
|
||||
f'/var/run/{os.getuid()}',
|
||||
f'/run/user/{os.getuid()}',
|
||||
),
|
||||
'.',
|
||||
'borgmatic',
|
||||
)
|
||||
)
|
||||
@ -49,14 +52,13 @@ def get_borgmatic_runtime_directory(config):
|
||||
def get_borgmatic_state_directory(config):
|
||||
'''
|
||||
Given a configuration dict, get the borgmatic state directory used for storing borgmatic state
|
||||
files like records of when checks last ran. Defaults to the "borgmatic_source_directory" value
|
||||
(deprecated) or $XDG_STATE_HOME/borgmatic or ~/.local/state/borgmatic.
|
||||
files like records of when checks last ran. Defaults to $XDG_STATE_HOME/borgmatic or
|
||||
~/.local/state/./borgmatic.
|
||||
'''
|
||||
return expand_user_in_path(
|
||||
config.get('borgmatic_state_directory')
|
||||
or config.get('borgmatic_source_directory')
|
||||
or os.path.join(
|
||||
os.environ.get(
|
||||
os.path.join(
|
||||
config.get('user_state_directory')
|
||||
or os.environ.get(
|
||||
'XDG_STATE_HOME',
|
||||
'~/.local/state',
|
||||
),
|
||||
|
||||
@ -207,24 +207,27 @@ properties:
|
||||
borgmatic_source_directory:
|
||||
type: string
|
||||
description: |
|
||||
Deprecated. Replaced by borgmatic_runtime_directory and
|
||||
Deprecated. Only used for locating database dumps and bootstrap
|
||||
metadata within backup archives created prior to deprecation.
|
||||
Replaced by borgmatic_runtime_directory and
|
||||
borgmatic_state_directory. Defaults to ~/.borgmatic
|
||||
example: /tmp/borgmatic
|
||||
borgmatic_runtime_directory:
|
||||
user_runtime_directory:
|
||||
type: string
|
||||
description: |
|
||||
Path for storing temporary runtime data like streaming database
|
||||
dumps and bootstrap metadata. Defaults to the
|
||||
borgmatic_source_directory value (deprecated) or
|
||||
$XDG_RUNTIME_DIR/borgmatic or /var/run/$UID/borgmatic.
|
||||
dumps and bootstrap metadata. borgmatic automatically creates and
|
||||
uses a "borgmatic" subdirectory here. Defaults to $XDG_RUNTIME_DIR
|
||||
or /run/user/$UID.
|
||||
example: /run/user/1001/borgmatic
|
||||
borgmatic_state_directory:
|
||||
user_state_directory:
|
||||
type: string
|
||||
description: |
|
||||
Path for storing borgmatic state files like records of when checks
|
||||
last ran. Defaults to the borgmatic_source_directory value
|
||||
(deprecated) or $XDG_STATE_HOME/borgmatic or
|
||||
~/.local/state/borgmatic.
|
||||
last ran. borgmatic automatically creates and uses a "borgmatic"
|
||||
subdirectory here. If you change this option, borgmatic must
|
||||
create the check records again (and therefore re-run checks).
|
||||
Defaults to $XDG_STATE_HOME or ~/.local/state.
|
||||
example: /var/lib/borgmatic
|
||||
store_config_files:
|
||||
type: boolean
|
||||
|
||||
@ -1,9 +1,8 @@
|
||||
import fnmatch
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DATA_SOURCE_HOOK_NAMES = (
|
||||
@ -15,15 +14,12 @@ DATA_SOURCE_HOOK_NAMES = (
|
||||
)
|
||||
|
||||
|
||||
def make_data_source_dump_path(borgmatic_source_directory, data_source_hook_name):
|
||||
def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_name):
|
||||
'''
|
||||
Given a borgmatic source directory (or None) and a data source hook name, construct a data
|
||||
source dump path.
|
||||
Given a borgmatic runtime directory and a data source hook name, construct a data source dump
|
||||
path.
|
||||
'''
|
||||
if not borgmatic_source_directory:
|
||||
borgmatic_source_directory = DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||
|
||||
return os.path.join(borgmatic_source_directory, data_source_hook_name)
|
||||
return os.path.join(borgmatic_runtime_directory, data_source_hook_name)
|
||||
|
||||
|
||||
def make_data_source_dump_filename(dump_path, name, hostname=None):
|
||||
@ -36,7 +32,7 @@ def make_data_source_dump_filename(dump_path, name, hostname=None):
|
||||
if os.path.sep in name:
|
||||
raise ValueError(f'Invalid data source name {name}')
|
||||
|
||||
return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
|
||||
return os.path.join(dump_path, hostname or 'localhost', name)
|
||||
|
||||
|
||||
def create_parent_directory_for_dump(dump_path):
|
||||
@ -63,18 +59,22 @@ def remove_data_source_dumps(dump_path, data_source_type_name, log_prefix, dry_r
|
||||
|
||||
logger.debug(f'{log_prefix}: Removing {data_source_type_name} data source dumps{dry_run_label}')
|
||||
|
||||
expanded_path = os.path.expanduser(dump_path)
|
||||
|
||||
if dry_run:
|
||||
return
|
||||
|
||||
if os.path.exists(expanded_path):
|
||||
shutil.rmtree(expanded_path)
|
||||
if os.path.exists(dump_path):
|
||||
shutil.rmtree(dump_path)
|
||||
|
||||
|
||||
def convert_glob_patterns_to_borg_patterns(patterns):
|
||||
def convert_glob_patterns_to_borg_pattern(patterns):
|
||||
'''
|
||||
Convert a sequence of shell glob patterns like "/etc/*" to the corresponding Borg archive
|
||||
patterns like "sh:etc/*".
|
||||
Convert a sequence of shell glob patterns like "/etc/*", "/tmp/*" to the corresponding Borg
|
||||
regular expression archive pattern as a single string like "re:etc/.*|tmp/.*".
|
||||
'''
|
||||
return [f'sh:{pattern.lstrip(os.path.sep)}' for pattern in patterns]
|
||||
# Remove the "\Z" generated by fnmatch.translate() because we don't want the pattern to match
|
||||
# only at the end of a path, as directory format dumps require extracting files with paths
|
||||
# longer than the pattern. E.g., a pattern of "borgmatic/*/foo_databases/test" should also match
|
||||
# paths like "borgmatic/*/foo_databases/test/toc.dat"
|
||||
return 're:' + '|'.join(
|
||||
fnmatch.translate(pattern.lstrip('/')).replace('\\Z', '') for pattern in patterns
|
||||
)
|
||||
|
||||
@ -3,6 +3,7 @@ import logging
|
||||
import os
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
from borgmatic.execute import (
|
||||
execute_command,
|
||||
execute_command_and_capture_output,
|
||||
@ -13,12 +14,14 @@ from borgmatic.hooks import dump
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_dump_path(config): # pragma: no cover
|
||||
def make_dump_path(config, base_directory=None): # pragma: no cover
|
||||
'''
|
||||
Make the dump path from the given configuration dict and the name of this hook.
|
||||
Given a configuration dict and an optional base directory, make the corresponding dump path. If
|
||||
a base directory isn't provided, use the borgmatic runtime directory.
|
||||
'''
|
||||
return dump.make_data_source_dump_path(
|
||||
config.get('borgmatic_source_directory'), 'mariadb_databases'
|
||||
base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config),
|
||||
'mariadb_databases',
|
||||
)
|
||||
|
||||
|
||||
@ -191,13 +194,23 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma:
|
||||
dump.remove_data_source_dumps(make_dump_path(config), 'MariaDB', log_prefix, dry_run)
|
||||
|
||||
|
||||
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
'''
|
||||
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
|
||||
database name to match, return the corresponding glob patterns to match the database dump in an
|
||||
archive.
|
||||
'''
|
||||
return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*')
|
||||
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
|
||||
|
||||
return (
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, 'borgmatic'), name, hostname='*'
|
||||
),
|
||||
dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'),
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, borgmatic_source_directory), name, hostname='*'
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def restore_data_source_dump(
|
||||
|
||||
@ -1,18 +1,21 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
from borgmatic.execute import execute_command, execute_command_with_processes
|
||||
from borgmatic.hooks import dump
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_dump_path(config): # pragma: no cover
|
||||
def make_dump_path(config, base_directory=None): # pragma: no cover
|
||||
'''
|
||||
Make the dump path from the given configuration dict and the name of this hook.
|
||||
Given a configuration dict and an optional base directory, make the corresponding dump path. If
|
||||
a base directory isn't provided, use the borgmatic runtime directory.
|
||||
'''
|
||||
return dump.make_data_source_dump_path(
|
||||
config.get('borgmatic_source_directory'), 'mongodb_databases'
|
||||
base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config),
|
||||
'mongodb_databases',
|
||||
)
|
||||
|
||||
|
||||
@ -100,13 +103,23 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma:
|
||||
dump.remove_data_source_dumps(make_dump_path(config), 'MongoDB', log_prefix, dry_run)
|
||||
|
||||
|
||||
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
'''
|
||||
Given a sequence of database configurations dicts, a configuration dict, a prefix to log with,
|
||||
and a database name to match, return the corresponding glob patterns to match the database dump
|
||||
in an archive.
|
||||
'''
|
||||
return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*')
|
||||
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
|
||||
|
||||
return (
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, 'borgmatic'), name, hostname='*'
|
||||
),
|
||||
dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'),
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, borgmatic_source_directory), name, hostname='*'
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def restore_data_source_dump(
|
||||
|
||||
@ -3,6 +3,7 @@ import logging
|
||||
import os
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
from borgmatic.execute import (
|
||||
execute_command,
|
||||
execute_command_and_capture_output,
|
||||
@ -13,12 +14,14 @@ from borgmatic.hooks import dump
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_dump_path(config): # pragma: no cover
|
||||
def make_dump_path(config, base_directory=None): # pragma: no cover
|
||||
'''
|
||||
Make the dump path from the given configuration dict and the name of this hook.
|
||||
Given a configuration dict and an optional base directory, make the corresponding dump path. If
|
||||
a base directory isn't provided, use the borgmatic runtime directory.
|
||||
'''
|
||||
return dump.make_data_source_dump_path(
|
||||
config.get('borgmatic_source_directory'), 'mysql_databases'
|
||||
base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config),
|
||||
'mysql_databases',
|
||||
)
|
||||
|
||||
|
||||
@ -189,13 +192,23 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma:
|
||||
dump.remove_data_source_dumps(make_dump_path(config), 'MySQL', log_prefix, dry_run)
|
||||
|
||||
|
||||
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
'''
|
||||
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
|
||||
database name to match, return the corresponding glob patterns to match the database dump in an
|
||||
archive.
|
||||
'''
|
||||
return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*')
|
||||
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
|
||||
|
||||
return (
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, 'borgmatic'), name, hostname='*'
|
||||
),
|
||||
dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'),
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, borgmatic_source_directory), name, hostname='*'
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def restore_data_source_dump(
|
||||
|
||||
@ -2,8 +2,10 @@ import csv
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
from borgmatic.execute import (
|
||||
execute_command,
|
||||
execute_command_and_capture_output,
|
||||
@ -14,12 +16,14 @@ from borgmatic.hooks import dump
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_dump_path(config): # pragma: no cover
|
||||
def make_dump_path(config, base_directory=None): # pragma: no cover
|
||||
'''
|
||||
Make the dump path from the given configuration dict and the name of this hook.
|
||||
Given a configuration dict and an optional base directory, make the corresponding dump path. If
|
||||
a base directory isn't provided, use the borgmatic runtime directory.
|
||||
'''
|
||||
return dump.make_data_source_dump_path(
|
||||
config.get('borgmatic_source_directory'), 'postgresql_databases'
|
||||
base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config),
|
||||
'postgresql_databases',
|
||||
)
|
||||
|
||||
|
||||
@ -215,13 +219,23 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma:
|
||||
dump.remove_data_source_dumps(make_dump_path(config), 'PostgreSQL', log_prefix, dry_run)
|
||||
|
||||
|
||||
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
'''
|
||||
Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
|
||||
database name to match, return the corresponding glob patterns to match the database dump in an
|
||||
archive.
|
||||
'''
|
||||
return dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*')
|
||||
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
|
||||
|
||||
return (
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, 'borgmatic'), name, hostname='*'
|
||||
),
|
||||
dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'),
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, borgmatic_source_directory), name, hostname='*'
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def restore_data_source_dump(
|
||||
@ -291,7 +305,7 @@ def restore_data_source_dump(
|
||||
if 'restore_options' in data_source
|
||||
else ()
|
||||
)
|
||||
+ (() if extract_process else (dump_filename,))
|
||||
+ (() if extract_process else (str(pathlib.Path(dump_filename)),))
|
||||
+ tuple(
|
||||
itertools.chain.from_iterable(('--schema', schema) for schema in data_source['schemas'])
|
||||
if data_source.get('schemas')
|
||||
|
||||
@ -2,18 +2,21 @@ import logging
|
||||
import os
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
from borgmatic.execute import execute_command, execute_command_with_processes
|
||||
from borgmatic.hooks import dump
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_dump_path(config): # pragma: no cover
|
||||
def make_dump_path(config, base_directory=None): # pragma: no cover
|
||||
'''
|
||||
Make the dump path from the given configuration dict and the name of this hook.
|
||||
Given a configuration dict and an optional base directory, make the corresponding dump path. If
|
||||
a base directory isn't provided, use the borgmatic runtime directory.
|
||||
'''
|
||||
return dump.make_data_source_dump_path(
|
||||
config.get('borgmatic_source_directory'), 'sqlite_databases'
|
||||
base_directory or borgmatic.config.paths.get_borgmatic_runtime_directory(config),
|
||||
'sqlite_databases',
|
||||
)
|
||||
|
||||
|
||||
@ -87,12 +90,22 @@ def remove_data_source_dumps(databases, config, log_prefix, dry_run): # pragma:
|
||||
dump.remove_data_source_dumps(make_dump_path(config), 'SQLite', log_prefix, dry_run)
|
||||
|
||||
|
||||
def make_data_source_dump_pattern(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
def make_data_source_dump_patterns(databases, config, log_prefix, name=None): # pragma: no cover
|
||||
'''
|
||||
Make a pattern that matches the given SQLite databases. The databases are supplied as a sequence
|
||||
of configuration dicts, as per the configuration schema.
|
||||
'''
|
||||
return dump.make_data_source_dump_filename(make_dump_path(config), name)
|
||||
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
|
||||
|
||||
return (
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, 'borgmatic'), name, hostname='*'
|
||||
),
|
||||
dump.make_data_source_dump_filename(make_dump_path(config), name, hostname='*'),
|
||||
dump.make_data_source_dump_filename(
|
||||
make_dump_path(config, borgmatic_source_directory), name, hostname='*'
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def restore_data_source_dump(
|
||||
|
||||
@ -63,14 +63,23 @@ dump formats, which can't stream and therefore do consume temporary disk
|
||||
space. Additionally, prior to borgmatic 1.5.3, all database dumps consumed
|
||||
temporary disk space.)
|
||||
|
||||
To support this, borgmatic creates temporary named pipes in `~/.borgmatic` by
|
||||
default. To customize this path, set the `borgmatic_source_directory` option
|
||||
in borgmatic's configuration.
|
||||
<span class="minilink minilink-addedin">New in version 1.9.0</span> To support
|
||||
this, borgmatic creates temporary streaming database dumps within
|
||||
`/run/user/$UID/borgmatic` by default (where `$UID` is the current user's ID).
|
||||
To customize the `/run/user/$UID` portion of this path, set the
|
||||
`user_runtime_directory` option in borgmatic's configuration. Alternatively,
|
||||
set the `XDG_RUNTIME_DIR` environment variable (often already set to
|
||||
`/run/user/$UID`).
|
||||
|
||||
Also note that using a database hook implicitly enables both the
|
||||
`read_special` and `one_file_system` configuration settings (even if they're
|
||||
disabled in your configuration) to support this dump and restore streaming.
|
||||
See Limitations below for more on this.
|
||||
<span class="minilink minilink-addedin">Prior to version 1.9.0</span>
|
||||
borgmatic created temporary streaming database dumps within the `~/.borgmatic`
|
||||
directory by default. At that time, the path was configurable by the
|
||||
`borgmatic_source_directory` configuration option (now deprecated).
|
||||
|
||||
Also note that using a database hook implicitly enables the
|
||||
`read_special` configuration option (even if it's disabled in your
|
||||
configuration) to support this dump and restore streaming. See Limitations
|
||||
below for more on this.
|
||||
|
||||
Here's a more involved example that connects to remote databases:
|
||||
|
||||
|
||||
@ -225,15 +225,26 @@ repository, at most once a month.
|
||||
|
||||
Unlike a real scheduler like cron, borgmatic only makes a best effort to run
|
||||
checks on the configured frequency. It compares that frequency with how long
|
||||
it's been since the last check for a given repository (as recorded in a file
|
||||
within `~/.borgmatic/checks`). If it hasn't been long enough, the check is
|
||||
skipped. And you still have to run `borgmatic check` (or `borgmatic` without
|
||||
actions) in order for checks to run, even when a `frequency` is configured!
|
||||
it's been since the last check for a given repository If it hasn't been long
|
||||
enough, the check is skipped. And you still have to run `borgmatic check` (or
|
||||
`borgmatic` without actions) in order for checks to run, even when a
|
||||
`frequency` is configured!
|
||||
|
||||
This also applies *across* configuration files that have the same repository
|
||||
configured. Make sure you have the same check frequency configured in each
|
||||
though—or the most frequently configured check will apply.
|
||||
|
||||
<span class="minilink minilink-addedin">New in version 1.9.0</span>To support
|
||||
this frequency logic, borgmatic records check timestamps within the
|
||||
`~/.local/state/borgmatic/checks` directory. To override the `~/.local/state`
|
||||
portion of this path, set the `user_state_directory` configuration option.
|
||||
Alternatively, set the `XDG_STATE_HOME` environment variable.
|
||||
|
||||
<span class="minilink minilink-addedin">Prior to version 1.9.0</span>
|
||||
borgmatic recorded check timestamps within the `~/.borgmatic` directory. At
|
||||
that time, the path was configurable by the `borgmatic_source_directory`
|
||||
configuration option (now deprecated).
|
||||
|
||||
If you want to temporarily ignore your configured frequencies, you can invoke
|
||||
`borgmatic check --force` to run checks unconditionally.
|
||||
|
||||
|
||||
@ -62,9 +62,9 @@ for available values.
|
||||
|
||||
(No borgmatic `list` or `info` actions? Upgrade borgmatic!)
|
||||
|
||||
<span class="minilink minilink-addedin">New in borgmatic version 1.9.0</span>
|
||||
There are also `repo-list` and `repo-info` actions for displaying repository
|
||||
information with Borg 2.x:
|
||||
<span class="minilink minilink-addedin">New in version 1.9.0</span> There are
|
||||
also `repo-list` and `repo-info` actions for displaying repository information
|
||||
with Borg 2.x:
|
||||
|
||||
```bash
|
||||
borgmatic repo-list
|
||||
@ -107,12 +107,28 @@ hooks](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/), you
|
||||
can list backed up database dumps via borgmatic. For example:
|
||||
|
||||
```bash
|
||||
borgmatic list --archive latest --find .borgmatic/*_databases
|
||||
borgmatic list --archive latest --find *borgmatic/*_databases
|
||||
```
|
||||
|
||||
This gives you a listing of all database dump files contained in the latest
|
||||
archive, complete with file sizes.
|
||||
|
||||
<span class="minilink minilink-addedin">New in borgmatic version
|
||||
1.9.0</span>Database dump files are stored at `/borgmatic` within a backup
|
||||
archive, regardless of the user who performs the backup. (Note that Borg
|
||||
doesn't store the leading `/`.)
|
||||
|
||||
<span class="minilink minilink-addedin">With Borg version 1.2 and
|
||||
earlier</span>Database dump files are stored at `/var/run/$UID/borgmatic`
|
||||
(where `$UID` is the current user's ID) unless overridden by the
|
||||
`user_runtime_directory` configuration option or the `XDG_STATE_HOME`
|
||||
environment variable.
|
||||
|
||||
<span class="minilink minilink-addedin">Prior to borgmatic version
|
||||
1.9.0</span>Database dump files were instead stored at `~/.borgmatic` within
|
||||
the backup archive (where `~` was expanded to the home directory of the user
|
||||
who performed the backup). This applied with all versions of Borg.
|
||||
|
||||
|
||||
## Logging
|
||||
|
||||
|
||||
@ -22,7 +22,7 @@ following, all based on your borgmatic configuration files or command-line
|
||||
arguments:
|
||||
|
||||
* configured repositories, running your Borg command once for each one
|
||||
* local and remote Borg binary paths
|
||||
* local and remote Borg executable paths
|
||||
* SSH settings and Borg environment variables
|
||||
* lock wait settings
|
||||
* verbosity
|
||||
|
||||
@ -14,7 +14,7 @@ def write_configuration(
|
||||
source_directory,
|
||||
config_path,
|
||||
repository_path,
|
||||
borgmatic_source_directory,
|
||||
user_runtime_directory,
|
||||
postgresql_dump_format='custom',
|
||||
mongodb_dump_format='archive',
|
||||
):
|
||||
@ -28,7 +28,7 @@ source_directories:
|
||||
- {source_directory}
|
||||
repositories:
|
||||
- path: {repository_path}
|
||||
borgmatic_source_directory: {borgmatic_source_directory}
|
||||
user_runtime_directory: {user_runtime_directory}
|
||||
|
||||
encryption_passphrase: "test"
|
||||
|
||||
@ -101,7 +101,7 @@ def write_custom_restore_configuration(
|
||||
source_directory,
|
||||
config_path,
|
||||
repository_path,
|
||||
borgmatic_source_directory,
|
||||
user_runtime_directory,
|
||||
postgresql_dump_format='custom',
|
||||
mongodb_dump_format='archive',
|
||||
):
|
||||
@ -115,7 +115,7 @@ source_directories:
|
||||
- {source_directory}
|
||||
repositories:
|
||||
- path: {repository_path}
|
||||
borgmatic_source_directory: {borgmatic_source_directory}
|
||||
user_runtime_directory: {user_runtime_directory}
|
||||
|
||||
encryption_passphrase: "test"
|
||||
|
||||
@ -173,7 +173,7 @@ def write_simple_custom_restore_configuration(
|
||||
source_directory,
|
||||
config_path,
|
||||
repository_path,
|
||||
borgmatic_source_directory,
|
||||
user_runtime_directory,
|
||||
postgresql_dump_format='custom',
|
||||
):
|
||||
'''
|
||||
@ -187,7 +187,7 @@ source_directories:
|
||||
- {source_directory}
|
||||
repositories:
|
||||
- path: {repository_path}
|
||||
borgmatic_source_directory: {borgmatic_source_directory}
|
||||
user_runtime_directory: {user_runtime_directory}
|
||||
|
||||
encryption_passphrase: "test"
|
||||
|
||||
@ -347,7 +347,6 @@ def test_database_dump_and_restore():
|
||||
# Create a Borg repository.
|
||||
temporary_directory = tempfile.mkdtemp()
|
||||
repository_path = os.path.join(temporary_directory, 'test.borg')
|
||||
borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
|
||||
|
||||
# Write out a special file to ensure that it gets properly excluded and Borg doesn't hang on it.
|
||||