Apply the "working_directory" option to all actions, not just "create". Also fix the glob expansion of "source_directories" values to respect the "working_directory" option (#609).
All checks were successful
build / test (push) Successful in 5m44s
build / docs (push) Successful in 1m36s

This commit is contained in:
Dan Helfman 2024-10-20 16:04:41 -07:00
parent c71da46963
commit bd4c672382
48 changed files with 1608 additions and 178 deletions

4
NEWS
View File

@ -1,4 +1,8 @@
1.9.0.dev0
* #609: Fix the glob expansion of "source_directories" values to respect the "working_directory"
option.
* #609: BREAKING: Apply the "working_directory" option to all actions, not just "create". This
includes repository paths, destination paths, mount points, etc.
* #914: Fix a confusing apparent hang when when the repository location changes, and instead
show a helpful error message.
* #915: BREAKING: Rename repository actions like "rcreate" to more explicit names like

View File

@ -14,6 +14,7 @@ import borgmatic.borg.extract
import borgmatic.borg.list
import borgmatic.borg.repo_list
import borgmatic.borg.state
import borgmatic.config.options
import borgmatic.config.validate
import borgmatic.execute
import borgmatic.hooks.command
@ -356,17 +357,13 @@ def collect_spot_check_source_paths(
)
)
borg_environment = borgmatic.borg.environment.make_environment(config)
try:
working_directory = os.path.expanduser(config.get('working_directory'))
except TypeError:
working_directory = None
working_directory = borgmatic.config.options.get_working_directory(config)
paths_output = borgmatic.execute.execute_command_and_capture_output(
create_flags + create_positional_arguments,
capture_stderr=True,
working_directory=working_directory,
extra_environment=borg_environment,
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
@ -377,7 +374,9 @@ def collect_spot_check_source_paths(
if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
)
return tuple(path for path in paths if os.path.isfile(path))
return tuple(
path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
)
BORG_DIRECTORY_FILE_TYPE = 'd'
@ -444,8 +443,11 @@ def compare_spot_check_hashes(
int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
)
source_sample_paths = tuple(random.sample(source_paths, sample_count))
working_directory = borgmatic.config.options.get_working_directory(config)
existing_source_sample_paths = {
source_path for source_path in source_sample_paths if os.path.exists(source_path)
source_path
for source_path in source_sample_paths
if os.path.exists(os.path.join(working_directory or '', source_path))
}
logger.debug(
f'{log_label}: Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
@ -469,7 +471,8 @@ def compare_spot_check_hashes(
(spot_check_config.get('xxh64sum_command', 'xxh64sum'),)
+ tuple(
path for path in source_sample_paths_subset if path in existing_source_sample_paths
)
),
working_directory=working_directory,
)
source_hashes.update(

View File

@ -2,6 +2,7 @@ import logging
import shlex
import borgmatic.commands.arguments
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -67,6 +68,7 @@ def run_arbitrary_borg(
'ARCHIVE': archive if archive else '',
},
),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
from borgmatic.borg import environment, flags
from borgmatic.execute import execute_command
@ -37,6 +38,7 @@ def break_lock(
execute_command(
full_command,
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.execute
import borgmatic.logger
from borgmatic.borg import environment, flags
@ -56,6 +57,7 @@ def change_passphrase(
output_file=borgmatic.execute.DO_NOT_CAPTURE,
output_log_level=logging.ANSWER,
extra_environment=environment.make_environment(config_without_passphrase),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -2,6 +2,7 @@ import argparse
import json
import logging
import borgmatic.config.options
from borgmatic.borg import environment, feature, flags, repo_info
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -167,6 +168,8 @@ def check_archives(
+ flags.make_repository_flags(repository_path, local_borg_version)
)
working_directory = borgmatic.config.options.get_working_directory(config)
# The Borg repair option triggers an interactive prompt, which won't work when output is
# captured. And progress messes with the terminal directly.
if check_arguments.repair or check_arguments.progress:
@ -174,6 +177,7 @@ def check_archives(
full_command,
output_file=DO_NOT_CAPTURE,
extra_environment=borg_environment,
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
@ -181,6 +185,7 @@ def check_archives(
execute_command(
full_command,
extra_environment=borg_environment,
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
from borgmatic.borg import environment, flags
from borgmatic.execute import execute_command
@ -49,6 +50,7 @@ def compact_segments(
full_command,
output_log_level=logging.INFO,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -7,6 +7,7 @@ import stat
import tempfile
import textwrap
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, feature, flags, state
from borgmatic.execute import (
@ -19,26 +20,28 @@ from borgmatic.execute import (
logger = logging.getLogger(__name__)
def expand_directory(directory):
def expand_directory(directory, working_directory):
'''
Given a directory path, expand any tilde (representing a user's home directory) and any globs
therein. Return a list of one or more resulting paths.
'''
expanded_directory = os.path.expanduser(directory)
expanded_directory = os.path.join(working_directory or '', os.path.expanduser(directory))
return glob.glob(expanded_directory) or [expanded_directory]
def expand_directories(directories):
def expand_directories(directories, working_directory=None):
'''
Given a sequence of directory paths, expand tildes and globs in each one. Return all the
resulting directories as a single flattened tuple.
Given a sequence of directory paths and an optional working directory, expand tildes and globs
in each one. Return all the resulting directories as a single flattened tuple.
'''
if directories is None:
return ()
return tuple(
itertools.chain.from_iterable(expand_directory(directory) for directory in directories)
itertools.chain.from_iterable(
expand_directory(directory, working_directory) for directory in directories
)
)
@ -53,17 +56,19 @@ def expand_home_directories(directories):
return tuple(os.path.expanduser(directory) for directory in directories)
def map_directories_to_devices(directories):
def map_directories_to_devices(directories, working_directory=None):
'''
Given a sequence of directories, return a map from directory to an identifier for the device on
which that directory resides or None if the path doesn't exist.
Given a sequence of directories and an optional working directory, return a map from directory
to an identifier for the device on which that directory resides or None if the path doesn't
exist.
This is handy for determining whether two different directories are on the same filesystem (have
the same device identifier).
'''
return {
directory: os.stat(directory).st_dev if os.path.exists(directory) else None
directory: os.stat(full_directory).st_dev if os.path.exists(full_directory) else None
for directory in directories
for full_directory in (os.path.join(working_directory or '', directory),)
}
@ -318,12 +323,8 @@ def check_all_source_directories_exist(source_directories, working_directory=Non
for source_directory in source_directories
if not all(
[
os.path.exists(directory)
for directory in expand_directory(
os.path.join(working_directory, source_directory)
if working_directory
else source_directory
)
os.path.exists(os.path.join(working_directory or '', directory))
for directory in expand_directory(source_directory, working_directory)
]
)
]
@ -356,10 +357,7 @@ def make_base_create_command(
(base Borg create command flags, Borg create command positional arguments, open pattern file
handle, open exclude file handle).
'''
try:
working_directory = os.path.expanduser(config.get('working_directory'))
except TypeError:
working_directory = None
working_directory = borgmatic.config.options.get_working_directory(config)
if config.get('source_directories_must_exist', False):
check_all_source_directories_exist(
@ -371,11 +369,15 @@ def make_base_create_command(
expand_directories(
tuple(config.get('source_directories', ()))
+ borgmatic_source_directories
+ tuple(config_paths if config.get('store_config_files', True) else ())
+ tuple(config_paths if config.get('store_config_files', True) else ()),
working_directory=working_directory,
)
),
additional_directory_devices=map_directories_to_devices(
expand_directories(pattern_root_directories(config.get('patterns')))
expand_directories(
pattern_root_directories(config.get('patterns')),
working_directory=working_directory,
)
),
)
@ -522,8 +524,11 @@ def create_archive(
create command while also triggering the given processes to produce output.
'''
borgmatic.logger.add_custom_log_levels()
working_directory = borgmatic.config.options.get_working_directory(config)
borgmatic_source_directories = expand_directories(
collect_borgmatic_source_directories(config.get('borgmatic_source_directory'))
collect_borgmatic_source_directories(config.get('borgmatic_source_directory')),
working_directory=working_directory,
)
(create_flags, create_positional_arguments, pattern_file, exclude_file) = (
@ -555,11 +560,6 @@ def create_archive(
# the terminal directly.
output_file = DO_NOT_CAPTURE if progress else None
try:
working_directory = os.path.expanduser(config.get('working_directory'))
except TypeError:
working_directory = None
borg_environment = environment.make_environment(config)
create_flags += (

View File

@ -5,6 +5,7 @@ import borgmatic.borg.environment
import borgmatic.borg.feature
import borgmatic.borg.flags
import borgmatic.borg.repo_delete
import borgmatic.config.options
import borgmatic.execute
logger = logging.getLogger(__name__)
@ -127,6 +128,7 @@ def delete_archives(
command,
output_log_level=logging.ANSWER,
extra_environment=borgmatic.borg.environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,6 +1,7 @@
import logging
import os
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -29,9 +30,10 @@ def export_key(
borgmatic.logger.add_custom_log_levels()
umask = config.get('umask', None)
lock_wait = config.get('lock_wait', None)
working_directory = borgmatic.config.options.get_working_directory(config)
if export_arguments.path and export_arguments.path != '-':
if os.path.exists(export_arguments.path):
if os.path.exists(os.path.join(working_directory or '', export_arguments.path)):
raise FileExistsError(
f'Destination path {export_arguments.path} already exists. Aborting.'
)
@ -66,6 +68,7 @@ def export_key(
output_file=output_file,
output_log_level=logging.ANSWER,
extra_environment=environment.make_environment(config),
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -70,6 +71,7 @@ def export_tar_archive(
output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
output_log_level=output_log_level,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -2,6 +2,7 @@ import logging
import os
import subprocess
import borgmatic.config.options
import borgmatic.config.validate
from borgmatic.borg import environment, feature, flags, repo_list
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -58,8 +59,8 @@ def extract_last_archive_dry_run(
execute_command(
full_extract_command,
working_directory=None,
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
@ -112,6 +113,8 @@ def extract_archive(
*(len(tuple(piece for piece in path.split(os.path.sep) if piece)) - 1 for path in paths)
)
working_directory = borgmatic.config.options.get_working_directory(config)
full_command = (
(local_path, 'extract')
+ (('--remote-path', remote_path) if remote_path else ())
@ -126,9 +129,13 @@ def extract_archive(
+ (('--progress',) if progress else ())
+ (('--stdout',) if extract_to_stdout else ())
+ flags.make_repository_archive_flags(
# Make the repository path absolute so the working directory changes below don't
# prevent Borg from finding the repo.
borgmatic.config.validate.normalize_repository_path(repository),
# Make the repository path absolute so the destination directory
# used below via changing the working directory doesn't prevent
# Borg from finding the repo. But also apply the user's configured
# working directory (if any) to the repo path.
borgmatic.config.validate.normalize_repository_path(
os.path.join(working_directory or '', repository)
),
archive,
local_borg_version,
)
@ -137,6 +144,9 @@ def extract_archive(
borg_environment = environment.make_environment(config)
borg_exit_codes = config.get('borg_exit_codes')
full_destination_path = (
os.path.join(working_directory or '', destination_path) if destination_path else None
)
# The progress output isn't compatible with captured and logged output, as progress messes with
# the terminal directly.
@ -144,8 +154,8 @@ def extract_archive(
return execute_command(
full_command,
output_file=DO_NOT_CAPTURE,
working_directory=destination_path,
extra_environment=borg_environment,
working_directory=full_destination_path,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
@ -155,9 +165,9 @@ def extract_archive(
return execute_command(
full_command,
output_file=subprocess.PIPE,
working_directory=destination_path,
run_to_completion=False,
extra_environment=borg_environment,
working_directory=full_destination_path,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
@ -166,8 +176,8 @@ def extract_archive(
# if the restore paths don't exist in the archive.
execute_command(
full_command,
working_directory=destination_path,
extra_environment=borg_environment,
working_directory=full_destination_path,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)

View File

@ -1,6 +1,7 @@
import argparse
import logging
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import execute_command, execute_command_and_capture_output
@ -96,10 +97,12 @@ def display_archives_info(
remote_path,
)
borg_exit_codes = config.get('borg_exit_codes')
working_directory = borgmatic.config.options.get_working_directory(config)
json_info = execute_command_and_capture_output(
json_command,
extra_environment=environment.make_environment(config),
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
@ -113,6 +116,7 @@ def display_archives_info(
main_command,
output_log_level=logging.ANSWER,
extra_environment=environment.make_environment(config),
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)

View File

@ -3,6 +3,7 @@ import copy
import logging
import re
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, feature, flags, repo_list
from borgmatic.execute import execute_command, execute_command_and_capture_output
@ -127,6 +128,7 @@ def capture_archive_listing(
remote_path,
),
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
@ -224,6 +226,7 @@ def list_archive(
remote_path,
),
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
@ -259,6 +262,7 @@ def list_archive(
main_command,
output_log_level=logging.ANSWER,
extra_environment=borg_environment,
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -59,6 +60,7 @@ def mount_archive(
)
borg_environment = environment.make_environment(config)
working_directory = borgmatic.config.options.get_working_directory(config)
# Don't capture the output when foreground mode is used so that ctrl-C can work properly.
if mount_arguments.foreground:
@ -66,6 +68,7 @@ def mount_archive(
full_command,
output_file=DO_NOT_CAPTURE,
extra_environment=borg_environment,
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
@ -74,6 +77,7 @@ def mount_archive(
execute_command(
full_command,
extra_environment=borg_environment,
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import execute_command
@ -95,6 +96,7 @@ def prune_archives(
full_command,
output_log_level=output_log_level,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -3,6 +3,7 @@ import json
import logging
import subprocess
import borgmatic.config.options
from borgmatic.borg import environment, feature, flags, repo_info
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -96,6 +97,7 @@ def create_repository(
repo_create_command,
output_file=DO_NOT_CAPTURE,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -3,6 +3,7 @@ import logging
import borgmatic.borg.environment
import borgmatic.borg.feature
import borgmatic.borg.flags
import borgmatic.config.options
import borgmatic.execute
logger = logging.getLogger(__name__)
@ -87,6 +88,7 @@ def delete_repository(
else borgmatic.execute.DO_NOT_CAPTURE
),
extra_environment=borgmatic.borg.environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import execute_command, execute_command_and_capture_output
@ -49,12 +50,14 @@ def display_repository_info(
)
extra_environment = environment.make_environment(config)
working_directory = borgmatic.config.options.get_working_directory(config)
borg_exit_codes = config.get('borg_exit_codes')
if repo_info_arguments.json:
return execute_command_and_capture_output(
full_command,
extra_environment=extra_environment,
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
@ -63,6 +66,7 @@ def display_repository_info(
full_command,
output_log_level=logging.ANSWER,
extra_environment=extra_environment,
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)

View File

@ -1,6 +1,7 @@
import argparse
import logging
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, feature, flags
from borgmatic.execute import execute_command, execute_command_and_capture_output
@ -48,6 +49,7 @@ def resolve_archive_name(
output = execute_command_and_capture_output(
full_command,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
@ -156,11 +158,13 @@ def list_repository(
local_path,
remote_path,
)
working_directory = borgmatic.config.options.get_working_directory(config)
borg_exit_codes = config.get('borg_exit_codes')
json_listing = execute_command_and_capture_output(
json_command,
extra_environment=borg_environment,
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
@ -174,6 +178,7 @@ def list_repository(
main_command,
output_log_level=logging.ANSWER,
extra_environment=borg_environment,
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
import borgmatic.logger
from borgmatic.borg import environment, flags
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@ -55,7 +56,8 @@ def transfer_archives(
full_command,
output_log_level=logging.ANSWER,
output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
extra_environment=environment.make_environment(config),
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
from borgmatic.execute import execute_command
logger = logging.getLogger(__name__)
@ -18,5 +19,8 @@ def unmount_archive(config, mount_point, local_path='borg'):
)
execute_command(
full_command, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes')
full_command,
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -1,5 +1,6 @@
import logging
import borgmatic.config.options
from borgmatic.borg import environment
from borgmatic.execute import execute_command_and_capture_output
@ -21,6 +22,7 @@ def local_borg_version(config, local_path='borg'):
output = execute_command_and_capture_output(
full_command,
extra_environment=environment.make_environment(config),
working_directory=borgmatic.config.options.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)

View File

@ -0,0 +1,11 @@
import os
def get_working_directory(config):
'''
Given a configuration dict, get the working directory from it, first expanding any tildes.
'''
try:
return os.path.expanduser(config.get('working_directory', '')) or None
except TypeError:
return None

View File

@ -58,9 +58,10 @@ properties:
working_directory:
type: string
description: |
Working directory for the "borg create" command. Tildes are
expanded. Useful for backing up using relative paths. See
http://borgbackup.readthedocs.io/en/stable/usage/create.html for
Working directory to use when running actions, useful for backing up
using relative source directory paths. Does not currently apply to
borgmatic configuration file paths or includes. Tildes are expanded.
See http://borgbackup.readthedocs.io/en/stable/usage/create.html for
details. Defaults to not set.
example: /path/to/working/directory
one_file_system:

View File

@ -497,6 +497,9 @@ def test_collect_spot_check_source_paths_parses_borg_output():
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
@ -534,6 +537,9 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
@ -571,6 +577,9 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
@ -608,6 +617,9 @@ def test_collect_spot_check_source_paths_skips_directories():
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
@ -668,14 +680,60 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_dire
) == ('/etc/path',)
def test_collect_spot_check_source_paths_uses_working_directory():
flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
{'hook1': False, 'hook2': True}
)
flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
dry_run=True,
repository_path='repo',
config=object,
config_paths=(),
local_borg_version=object,
global_arguments=object,
borgmatic_source_directories=(),
local_path=object,
remote_path=object,
list_files=True,
stream_processes=True,
).and_return((('borg', 'create'), ('repo::archive',), flexmock(), flexmock()))
flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
flexmock()
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
'/working/dir'
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
'warning: stuff\n- foo\n+ bar\n? /nope',
)
flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/foo').and_return(True)
flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/bar').and_return(True)
assert module.collect_spot_check_source_paths(
repository={'path': 'repo'},
config={'working_directory': '/working/dir'},
local_borg_version=flexmock(),
global_arguments=flexmock(),
local_path=flexmock(),
remote_path=flexmock(),
) == ('foo', 'bar')
def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('xxh64sum', '/foo', '/bar')).and_return('hash1 /foo\nhash2 /bar')
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
'hash1 /foo\nhash2 /bar'
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 /foo', 'nothash2 /bar']
)
@ -708,10 +766,15 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('xxh64sum', '/foo', '/bar')).and_return('hash1 /foo\nhash2 /bar')
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
'hash1 /foo\nhash2 /bar'
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['nothash1 /foo', 'nothash2 /bar']
)
@ -744,10 +807,15 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('/usr/local/bin/xxh64sum', '/foo', '/bar')).and_return('hash1 /foo\nhash2 /bar')
).with_args(('/usr/local/bin/xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
'hash1 /foo\nhash2 /bar'
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 /foo', 'nothash2 /bar']
)
@ -773,14 +841,19 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
) == ('/bar',)
def test_compare_spot_check_hashes_consider_path_missing_from_archive_as_not_matching():
def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_matching():
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('xxh64sum', '/foo', '/bar')).and_return('hash1 /foo\nhash2 /bar')
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
'hash1 /foo\nhash2 /bar'
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 /foo']
)
@ -809,11 +882,14 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching()
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').with_args('/foo').and_return(True)
flexmock(module.os.path).should_receive('exists').with_args('/bar').and_return(False)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('xxh64sum', '/foo')).and_return('hash1 /foo')
).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 /foo', 'hash2 /bar']
)
@ -843,13 +919,20 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('xxh64sum', '/foo', '/bar')).and_return('hash1 /foo\nhash2 /bar')
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
'hash1 /foo\nhash2 /bar'
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('xxh64sum', '/baz', '/quux')).and_return('hash3 /baz\nhash4 /quux')
).with_args(('xxh64sum', '/baz', '/quux'), working_directory=None).and_return(
'hash3 /baz\nhash4 /quux'
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 /foo', 'hash2 /bar']
).and_return(['hash3 /baz', 'nothash4 /quux'])
@ -878,6 +961,49 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in
) == ('/quux',)
def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths():
flexmock(module.random).should_receive('sample').replace_with(
lambda population, count: population[:count]
)
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
'/working/dir',
)
flexmock(module.os.path).should_receive('exists').with_args('/working/dir/foo').and_return(True)
flexmock(module.os.path).should_receive('exists').with_args('/working/dir/bar').and_return(True)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('xxh64sum', 'foo', 'bar'), working_directory='/working/dir').and_return(
'hash1 foo\nhash2 bar'
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 foo', 'nothash2 bar']
)
assert module.compare_spot_check_hashes(
repository={'path': 'repo'},
archive='archive',
config={
'checks': [
{
'name': 'archives',
'frequency': '2 weeks',
},
{
'name': 'spot',
'data_sample_percentage': 50,
},
],
'working_directory': '/working/dir',
},
local_borg_version=flexmock(),
global_arguments=flexmock(),
local_path=flexmock(),
remote_path=flexmock(),
log_label='repo',
source_paths=('foo', 'bar', 'baz', 'quux'),
) == ('bar',)
def test_spot_check_without_spot_configuration_errors():
with pytest.raises(ValueError):
module.spot_check(

View File

@ -12,13 +12,17 @@ def test_run_arbitrary_borg_calls_borg_with_flags():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', '::'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
borg_exit_codes=None,
shell=True,
extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
)
module.run_arbitrary_borg(
@ -34,13 +38,17 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', '--info', '::'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
borg_exit_codes=None,
shell=True,
extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
)
insert_logging_mock(logging.INFO)
@ -57,13 +65,17 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', '--debug', '--show-rc', '::'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
borg_exit_codes=None,
shell=True,
extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
)
insert_logging_mock(logging.DEBUG)
@ -83,13 +95,17 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags():
('--lock-wait', '5')
)
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', '--lock-wait', '5', '::'),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
borg_exit_codes=None,
shell=True,
extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
)
module.run_arbitrary_borg(
@ -105,13 +121,17 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module).should_receive('execute_command').with_args(
('borg', 'break-lock', "'::$ARCHIVE'"),
output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
borg_local_path='borg',
borg_exit_codes=None,
shell=True,
extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'},
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
)
module.run_arbitrary_borg(
@ -128,13 +148,17 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
flexmock(module.flags).should_receive('make_flags').and_return(())
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.options).should_receive('get_working_directory').and_return(
None
)
flexmock(module).should_receive('execute_command').with_args(
('borg1', 'break-lock', '::'),